Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix the case where RTR does not clean up a completed task on startup

This commit is contained in:
fjy 2013-08-15 13:09:02 -07:00
parent 0c5a906a1b
commit 1fb6107a37
4 changed files with 65 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
indexing-service/src/main/java/com/metamx/druid/indexing/coordinator/RemoteTaskRunner.java
View File

@ -283,14 +283,26 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider
{
RemoteTaskRunnerWorkItem runningTask = runningTasks.get(task.getId());
if (runningTask != null) {
log.info("Assigned a task[%s] that is already running, not doing anything", task.getId());
ZkWorker zkWorker = findWorkerRunningTask(task.getId());
if (zkWorker == null) {
log.error("Got task %s that is running but no worker is actually running it?", task.getId());
} else {
log.info("Task[%s] already running on %s.", task.getId(), zkWorker.getWorker().getHost());
TaskStatus status = zkWorker.getRunningTasks().get(task.getId());
if (status.isComplete()) {
taskComplete(runningTask, zkWorker, task.getId(), status);
}
}
return runningTask.getResult();
}
RemoteTaskRunnerWorkItem pendingTask = pendingTasks.get(task.getId());
if (pendingTask != null) {
log.info("Assigned a task[%s] that is already pending, not doing anything", task.getId());
return pendingTask.getResult();
}
RemoteTaskRunnerWorkItem taskRunnerWorkItem = new RemoteTaskRunnerWorkItem(
task,
SettableFuture.<TaskStatus>create()
@ -473,8 +485,8 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider
}
catch (Exception e) {
log.makeAlert("Exception while trying to run task")
.addData("taskId", taskRunnerWorkItem.getTask().getId())
.emit();
.addData("taskId", taskRunnerWorkItem.getTask().getId())
.emit();
}
}
@ -590,16 +602,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider
}
if (taskStatus.isComplete()) {
if (taskRunnerWorkItem != null) {
final ListenableFuture<TaskStatus> result = taskRunnerWorkItem.getResult();
if (result != null) {
((SettableFuture<TaskStatus>) result).set(taskStatus);
}
}
// Worker is done with this task
zkWorker.setLastCompletedTaskTime(new DateTime());
cleanup(zkWorker.getWorker().getHost(), taskId);
taskComplete(taskRunnerWorkItem, zkWorker, taskId, taskStatus);
runPendingTasks();
}
break;
@ -710,4 +713,23 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider
log.debug("Worker nodes %s do not have capacity to run any more tasks!", zkWorkers.values());
return null;
}
private void taskComplete(
RemoteTaskRunnerWorkItem taskRunnerWorkItem,
ZkWorker zkWorker,
String taskId,
TaskStatus taskStatus
)
{
if (taskRunnerWorkItem != null) {
final ListenableFuture<TaskStatus> result = taskRunnerWorkItem.getResult();
if (result != null) {
((SettableFuture<TaskStatus>) result).set(taskStatus);
}
}
// Worker is done with this task
zkWorker.setLastCompletedTaskTime(new DateTime());
cleanup(zkWorker.getWorker().getHost(), taskId);
}
}

7
indexing-service/src/main/java/com/metamx/druid/indexing/coordinator/ZkWorker.java
View File

@ -39,6 +39,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/**
* Holds information about a worker and a listener for task status changes associated with the worker.
@ -49,7 +50,7 @@ public class ZkWorker implements Closeable
private final PathChildrenCache statusCache;
private final Function<ChildData, TaskStatus> cacheConverter;
private volatile DateTime lastCompletedTaskTime = new DateTime();
private AtomicReference<DateTime> lastCompletedTaskTime = new AtomicReference<DateTime>(new DateTime());
public ZkWorker(Worker worker, PathChildrenCache statusCache, final ObjectMapper jsonMapper)
{
@ -128,7 +129,7 @@ public class ZkWorker implements Closeable
@JsonProperty
public DateTime getLastCompletedTaskTime()
{
return lastCompletedTaskTime;
return lastCompletedTaskTime.get();
}
public boolean isRunningTask(String taskId)
@ -154,7 +155,7 @@ public class ZkWorker implements Closeable
public void setLastCompletedTaskTime(DateTime completedTaskTime)
{
lastCompletedTaskTime = completedTaskTime;
lastCompletedTaskTime.getAndSet(completedTaskTime);
}
@Override

19
indexing-service/src/test/java/com/metamx/druid/indexing/coordinator/RemoteTaskRunnerTest.java
View File

@ -258,6 +258,25 @@ public class RemoteTaskRunnerTest
Assert.assertFalse(runningTasks.contains("first"));
}
@Test
public void testRunWithTaskComplete() throws Exception
{
cf.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.EPHEMERAL)
.forPath(joiner.join(statusPath, task.getId()), jsonMapper.writeValueAsBytes(TaskStatus.success(task.getId())));
doSetup();
remoteTaskRunner.bootstrap(Arrays.<Task>asList(task));
ListenableFuture<TaskStatus> future = remoteTaskRunner.run(task);
TaskStatus status = future.get();
Assert.assertEquals(TaskStatus.Status.SUCCESS, status.getStatusCode());
}
private void doSetup() throws Exception
{
makeWorker();

33
server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java
View File

@ -47,9 +47,8 @@ public class CostBalancerStrategy implements BalancerStrategy
DataSegment proposalSegment, List<ServerHolder> serverHolders
)
{
ServerHolder holder= chooseBestServer(proposalSegment, serverHolders, false).rhs;
if (holder!=null && !holder.isServingSegment(proposalSegment))
{
ServerHolder holder = chooseBestServer(proposalSegment, serverHolders, false).rhs;
if (holder != null && !holder.isServingSegment(proposalSegment)) {
return holder;
}
return null;
@ -65,7 +64,6 @@ public class CostBalancerStrategy implements BalancerStrategy
}
/**
* For assignment, we want to move to the lowest cost server that isn't already serving the segment.
*
@ -83,25 +81,10 @@ public class CostBalancerStrategy implements BalancerStrategy
{
Pair<Double, ServerHolder> bestServer = Pair.of(Double.POSITIVE_INFINITY, null);
MinMaxPriorityQueue<Pair<Double, ServerHolder>> costsAndServers = MinMaxPriorityQueue.orderedBy(
new Comparator<Pair<Double, ServerHolder>>()
{
@Override
public int compare(
Pair<Double, ServerHolder> o,
Pair<Double, ServerHolder> o1
)
{
return Double.compare(o.lhs, o1.lhs);
}
}
).create();
final long proposalSegmentSize = proposalSegment.getSize();
for (ServerHolder server : serverHolders) {
if (includeCurrentServer || !server.isServingSegment(proposalSegment))
{
if (includeCurrentServer || !server.isServingSegment(proposalSegment)) {
/** Don't calculate cost if the server doesn't have enough space or is loading the segment */
if (proposalSegmentSize > server.getAvailableSize() || server.isLoadingSegment(proposalSegment)) {
continue;
@ -159,10 +142,10 @@ public class CostBalancerStrategy implements BalancerStrategy
referenceTimestamp.getMillis() - segment1.getInterval().getEndMillis(),
referenceTimestamp.getMillis() - segment2.getInterval().getEndMillis()
);
double segment1diff=referenceTimestamp.getMillis()-segment1.getInterval().getEndMillis();
double segment2diff=referenceTimestamp.getMillis()-segment2.getInterval().getEndMillis();
if (segment1diff < SEVEN_DAYS_IN_MILLIS && segment2diff <SEVEN_DAYS_IN_MILLIS) {
recencyPenalty = (2 - segment1diff / SEVEN_DAYS_IN_MILLIS)*(2-segment2diff /SEVEN_DAYS_IN_MILLIS);
double segment1diff = referenceTimestamp.getMillis() - segment1.getInterval().getEndMillis();
double segment2diff = referenceTimestamp.getMillis() - segment2.getInterval().getEndMillis();
if (segment1diff < SEVEN_DAYS_IN_MILLIS && segment2diff < SEVEN_DAYS_IN_MILLIS) {
recencyPenalty = (2 - segment1diff / SEVEN_DAYS_IN_MILLIS) * (2 - segment2diff / SEVEN_DAYS_IN_MILLIS);
}
/** gap is null if the two segment intervals overlap or if they're adjacent */
@ -251,6 +234,4 @@ public class CostBalancerStrategy implements BalancerStrategy
);
}
}