Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

autoscaling will clear state if a node takes too long to create

This commit is contained in:
Fangjin Yang 2013-03-13 14:28:18 -07:00
parent ac2d4e52da
commit 86c7ebe1e2
5 changed files with 55 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java
View File

@ -176,6 +176,9 @@ public class IndexerCoordinatorResource
if (!configManager.set(WorkerSetupData.CONFIG_KEY, workerSetupData)) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
log.info("Updating Worker Setup configs: %s", workerSetupData);
return Response.ok().build();
}

30
merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java
View File

@ -103,18 +103,21 @@ public class SimpleResourceManagementStrategy implements ResourceManagementStrat
}
} else {
Duration durSinceLastProvision = new Duration(lastProvisionTime, new DateTime());
if (durSinceLastProvision.isLongerThan(config.getMaxScalingDuration())) {
log.makeAlert("Worker node provisioning taking too long")
.addData("millisSinceLastProvision", durSinceLastProvision.getMillis())
.addData("provisioningCount", currentlyProvisioning.size())
.emit();
}
log.info(
"%s still provisioning. Wait for all provisioned nodes to complete before requesting new worker. Current wait time: %s",
currentlyProvisioning,
durSinceLastProvision
);
if (durSinceLastProvision.isLongerThan(config.getMaxScalingDuration())) {
log.makeAlert("Worker node provisioning taking too long!")
.addData("millisSinceLastProvision", durSinceLastProvision.getMillis())
.addData("provisioningCount", currentlyProvisioning.size())
.emit();
currentlyProvisioning.clear();
}
}
return false;
@ -205,17 +208,20 @@ public class SimpleResourceManagementStrategy implements ResourceManagementStrat
}
} else {
Duration durSinceLastTerminate = new Duration(lastTerminateTime, new DateTime());
if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration())) {
log.makeAlert("Worker node termination taking too long")
.addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis())
.addData("terminatingCount", currentlyTerminating.size())
.emit();
}
log.info(
"%s still terminating. Wait for all nodes to terminate before trying again.",
currentlyTerminating
);
if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration())) {
log.makeAlert("Worker node termination taking too long!")
.addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis())
.addData("terminatingCount", currentlyTerminating.size())
.emit();
currentlyTerminating.clear();
}
}
return false;

13
merger/src/main/java/com/metamx/druid/merger/coordinator/setup/EC2NodeData.java
View File

@ -91,4 +91,17 @@ public class EC2NodeData
{
return keyName;
}
@Override
public String toString()
{
return "EC2NodeData{" +
"amiId='" + amiId + '\'' +
", instanceType='" + instanceType + '\'' +
", minInstances=" + minInstances +
", maxInstances=" + maxInstances +
", securityGroupIds=" + securityGroupIds +
", keyName='" + keyName + '\'' +
'}';
}
}

10
merger/src/main/java/com/metamx/druid/merger/coordinator/setup/GalaxyUserData.java
View File

@ -60,4 +60,14 @@ public class GalaxyUserData
{
return type;
}
@Override
public String toString()
{
return "GalaxyUserData{" +
"env='" + env + '\'' +
", version='" + version + '\'' +
", type='" + type + '\'' +
'}';
}
}

11
merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java
View File

@ -75,4 +75,15 @@ public class WorkerSetupData
{
return userData;
}
@Override
public String toString()
{
return "WorkerSetupData{" +
"minVersion='" + minVersion + '\'' +
", minNumWorkers=" + minNumWorkers +
", nodeData=" + nodeData +
", userData=" + userData +
'}';
}
}