fix RemoteTaskRunner terminating lazy workers below autoscaler minNumWorkers value (#5310)

* fix RemoteTaskRunner terminating lazy workers below autoscaler minNumWorkers value

* add comment
This commit is contained in:
Clint Wylie 2018-02-01 08:57:01 -08:00 committed by Roman Leventov
parent 3a69b0e513
commit 1fffc681d2
2 changed files with 23 additions and 0 deletions

View File

@ -1235,6 +1235,11 @@ public class RemoteTaskRunner implements WorkerTaskRunner, TaskLogStreamer
@Override
public Collection<Worker> markWorkersLazy(Predicate<ImmutableWorkerInfo> isLazyWorker, int maxWorkers)
{
// skip the lock and bail early if we should not mark any workers lazy (e.g. number
// of current workers is at or below the minNumWorkers of autoscaler config)
if (maxWorkers < 1) {
return Collections.emptyList();
}
// status lock is used to prevent any tasks being assigned to the worker while we mark it lazy
synchronized (statusLock) {
Iterator<String> iterator = zkWorkers.keySet().iterator();

View File

@ -550,6 +550,24 @@ public class RemoteTaskRunnerTest
Assert.assertEquals(1, remoteTaskRunner.getLazyWorkers().size());
}
@Test
public void testFindLazyWorkerNotRunningAnyTaskButWithZeroMaxWorkers() throws Exception
{
doSetup();
Collection<Worker> lazyworkers = remoteTaskRunner.markWorkersLazy(
new Predicate<ImmutableWorkerInfo>()
{
@Override
public boolean apply(ImmutableWorkerInfo input)
{
return true;
}
}, 0
);
Assert.assertEquals(0, lazyworkers.size());
Assert.assertEquals(0, remoteTaskRunner.getLazyWorkers().size());
}
@Test
public void testWorkerZKReconnect() throws Exception
{