Fix for RoundRobinStorageLocationSelectorStrategy not to pick the same storage location over and again (#8634)

* Fix for 8614: iterators returned by strategy don't share iteration state.

* Adding comments
This commit is contained in:
Sashidhar Thallam 2019-10-12 07:23:52 +05:30 committed by Benedict Jin
parent 6c609293b2
commit 124efa85f6
2 changed files with 62 additions and 12 deletions

View File

@ -48,6 +48,9 @@ public class RoundRobinStorageLocationSelectorStrategy implements StorageLocatio
private final int numStorageLocations = storageLocations.size(); private final int numStorageLocations = storageLocations.size();
private int remainingIterations = numStorageLocations; private int remainingIterations = numStorageLocations;
// Each call to this methods starts with a different startIndex to avoid the same location being picked up over
// again. See https://github.com/apache/incubator-druid/issues/8614.
private int i = startIndex.getAndUpdate(n -> (n + 1) % numStorageLocations);
@Override @Override
public boolean hasNext() public boolean hasNext()
@ -62,8 +65,10 @@ public class RoundRobinStorageLocationSelectorStrategy implements StorageLocatio
throw new NoSuchElementException(); throw new NoSuchElementException();
} }
remainingIterations--; remainingIterations--;
final StorageLocation nextLocation = final StorageLocation nextLocation = storageLocations.get(i++);
storageLocations.get(startIndex.getAndUpdate(n -> (n + 1) % numStorageLocations)); if (i == numStorageLocations) {
i = 0;
}
return nextLocation; return nextLocation;
} }
}; };

View File

@ -110,16 +110,7 @@ public class StorageLocationSelectorStrategyTest
StorageLocationSelectorStrategy roundRobinStrategy = new RoundRobinStorageLocationSelectorStrategy(storageLocations); StorageLocationSelectorStrategy roundRobinStrategy = new RoundRobinStorageLocationSelectorStrategy(storageLocations);
iterateLocs(localStorageFolder1, localStorageFolder2, localStorageFolder3, roundRobinStrategy); // First call to getLocations()
iterateLocs(localStorageFolder1, localStorageFolder2, localStorageFolder3, roundRobinStrategy);
iterateLocs(localStorageFolder1, localStorageFolder2, localStorageFolder3, roundRobinStrategy);
iterateLocs(localStorageFolder1, localStorageFolder2, localStorageFolder3, roundRobinStrategy);
iterateLocs(localStorageFolder1, localStorageFolder2, localStorageFolder3, roundRobinStrategy);
}
private void iterateLocs(File localStorageFolder1, File localStorageFolder2, File localStorageFolder3,
StorageLocationSelectorStrategy roundRobinStrategy)
{
Iterator<StorageLocation> locations = roundRobinStrategy.getLocations(); Iterator<StorageLocation> locations = roundRobinStrategy.getLocations();
StorageLocation loc1 = locations.next(); StorageLocation loc1 = locations.next();
@ -133,6 +124,60 @@ public class StorageLocationSelectorStrategyTest
StorageLocation loc3 = locations.next(); StorageLocation loc3 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_3", Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_3",
localStorageFolder3, loc3.getPath()); localStorageFolder3, loc3.getPath());
// Second call to getLocations()
locations = roundRobinStrategy.getLocations();
loc2 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_2",
localStorageFolder2, loc2.getPath());
loc3 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_3",
localStorageFolder3, loc3.getPath());
loc1 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_1",
localStorageFolder1, loc1.getPath());
}
@Test
public void testRoundRobinLocationSelectorStrategyMultipleCallsToGetLocations() throws Exception
{
List<StorageLocation> storageLocations = new ArrayList<>();
final File localStorageFolder1 = tmpFolder.newFolder("local_storage_folder_1");
final File localStorageFolder2 = tmpFolder.newFolder("local_storage_folder_2");
final File localStorageFolder3 = tmpFolder.newFolder("local_storage_folder_3");
storageLocations.add(new StorageLocation(localStorageFolder1, 10000000000L, null));
storageLocations.add(new StorageLocation(localStorageFolder2, 10000000000L, null));
storageLocations.add(new StorageLocation(localStorageFolder3, 10000000000L, null));
StorageLocationSelectorStrategy roundRobinStrategy = new RoundRobinStorageLocationSelectorStrategy(storageLocations);
Iterator<StorageLocation> locations = roundRobinStrategy.getLocations();
StorageLocation loc1 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_1",
localStorageFolder1, loc1.getPath());
locations = roundRobinStrategy.getLocations();
StorageLocation loc2 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_2",
localStorageFolder2, loc2.getPath());
locations = roundRobinStrategy.getLocations();
StorageLocation loc3 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_3",
localStorageFolder3, loc3.getPath());
loc1 = locations.next();
Assert.assertEquals("The next element of the iterator should point to path local_storage_folder_1",
localStorageFolder1, loc1.getPath());
} }
} }