Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8390. Fix API incompatible changes in FairScheduler's AllocationFileLoaderService. (Gergo Repas via Haibo Chen)

This commit is contained in:
Haibo Chen 2018-06-04 09:14:53 -07:00
parent ab3885f2c8
commit ba12f87dcb
3 changed files with 58 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
View File

@ -87,7 +87,7 @@ public class AllocationFileLoaderService extends AbstractService {
private Path allocFile;
private FileSystem fs;
private final Listener reloadListener;
private Listener reloadListener;
@VisibleForTesting
long reloadIntervalMs = ALLOC_RELOAD_INTERVAL_MS;
@ -95,16 +95,15 @@ public class AllocationFileLoaderService extends AbstractService {
private Thread reloadThread;
private volatile boolean running = true;
public AllocationFileLoaderService(Listener reloadListener) {
this(reloadListener, SystemClock.getInstance());
public AllocationFileLoaderService() {
this(SystemClock.getInstance());
}
private List<Permission> defaultPermissions;
public AllocationFileLoaderService(Listener reloadListener, Clock clock) {
public AllocationFileLoaderService(Clock clock) {
super(AllocationFileLoaderService.class.getName());
this.clock = clock;
this.reloadListener = reloadListener;
}
@Override
@ -209,6 +208,10 @@ public class AllocationFileLoaderService extends AbstractService {
return allocPath;
}
public synchronized void setReloadListener(Listener reloadListener) {
this.reloadListener = reloadListener;
}
/**
* Updates the allocation list from the allocation config file. This file is
* expected to be in the XML format specified in the design doc.
@ -350,6 +353,7 @@ public class AllocationFileLoaderService extends AbstractService {
public interface Listener {
void onReload(AllocationConfiguration info) throws IOException;
void onCheck();
default void onCheck() {
}
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -208,8 +208,7 @@ public class FairScheduler extends
public FairScheduler() {
super(FairScheduler.class.getName());
context = new FSContext(this);
allocsLoader =
new AllocationFileLoaderService(new AllocationReloadListener());
allocsLoader = new AllocationFileLoaderService();
queueMgr = new QueueManager(this);
maxRunningEnforcer = new MaxRunningAppsEnforcer(this);
}
@ -1438,6 +1437,7 @@ public class FairScheduler extends
}
allocsLoader.init(conf);
allocsLoader.setReloadListener(new AllocationReloadListener());
// If we fail to load allocations file on initialize, we want to fail
// immediately. After a successful load, exceptions on future reloads
// will just result in leaving things as they are.

96
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
View File

@ -25,7 +25,6 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.Listener;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
@ -33,8 +32,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Fai
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
@ -82,8 +79,7 @@ public class TestAllocationFileLoaderService {
fs.copyFromLocalFile(new Path(fschedURL.toURI()), new Path(fsAllocPath));
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, fsAllocPath);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(Mockito.mock(Listener.class));
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
Path allocationFile = allocLoader.getAllocationFile(conf);
assertEquals(fsAllocPath, allocationFile.toString());
assertTrue(fs.exists(allocationFile));
@ -96,8 +92,7 @@ public class TestAllocationFileLoaderService {
throws UnsupportedFileSystemException {
Configuration conf = new YarnConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, "badfs:///badfile");
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(Mockito.mock(Listener.class));
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.getAllocationFile(conf);
}
@ -110,7 +105,7 @@ public class TestAllocationFileLoaderService {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
TEST_FAIRSCHED_XML);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(Mockito.mock(Listener.class));
new AllocationFileLoaderService();
Path allocationFile = allocLoader.getAllocationFile(conf);
assertEquals(TEST_FAIRSCHED_XML, allocationFile.getName());
assertTrue(fs.exists(allocationFile));
@ -139,11 +134,12 @@ public class TestAllocationFileLoaderService {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder, clock);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(
clock);
allocLoader.reloadIntervalMs = 5;
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration allocConf = confHolder.allocConf;
@ -209,9 +205,7 @@ public class TestAllocationFileLoaderService {
public void testAllocationFileParsing() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
AllocationFileWriter
.create()
@ -284,6 +278,8 @@ public class TestAllocationFileLoaderService {
.writeToFile(ALLOC_FILE);
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
@ -431,9 +427,7 @@ public class TestAllocationFileLoaderService {
public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@ -479,6 +473,8 @@ public class TestAllocationFileLoaderService {
out.close();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
@ -554,10 +550,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration allocConf = confHolder.allocConf;
@ -588,10 +584,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}
@ -612,10 +608,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}
@ -636,10 +632,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}
@ -658,10 +654,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
try {
allocLoader.reloadAllocations();
} catch (AllocationConfigurationException ex) {
@ -689,10 +685,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
try {
allocLoader.reloadAllocations();
} catch (AllocationConfigurationException ex) {
@ -718,10 +714,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
// Check whether queue 'parent' and 'child' are loaded successfully
@ -749,10 +745,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}
@ -771,10 +767,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}
@ -797,10 +793,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
AllocationConfiguration allocConf = confHolder.allocConf;
@ -857,10 +853,10 @@ public class TestAllocationFileLoaderService {
out.println("</allocations>");
out.close();
ReloadListener confHolder = new ReloadListener();
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(confHolder);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
allocLoader.setReloadListener(confHolder);
allocLoader.reloadAllocations();
}