Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke

This commit is contained in:
Szilard Nemeth 2021-12-14 22:00:43 +01:00
parent 364d38f00d
commit 843f66f4dc
6 changed files with 33 additions and 46 deletions

View File

@ -145,7 +145,8 @@ public abstract class AbstractCSQueue implements CSQueue {
CSQueueMetrics metrics = old != null ?
(CSQueueMetrics) old.getMetrics() :
CSQueueMetrics.forQueue(getQueuePath(), parent,
queueContext.getConfiguration().getEnableUserMetrics(), queueContext.getConfiguration());
queueContext.getConfiguration().getEnableUserMetrics(),
queueContext.getConfiguration());
this.usageTracker = new CSQueueUsageTracker(metrics);
this.queueCapacities = new QueueCapacities(parent == null);

View File

@ -453,8 +453,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
public List<QueueUserACLInfo>
getQueueUserAclInfo(UserGroupInformation user) {
public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) {
readLock.lock();
try {
QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
@ -527,8 +526,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws
IOException {
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
writeLock.lock();
try {
@ -634,7 +633,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for queues
//TODO recalculate max applications because they can depend on capacity
if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) {
if (getNumApplications() >= getMaxApplications() &&
!(this instanceof AutoCreatedLeafQueue)) {
String msg =
"Queue " + getQueuePath() + " already has " + getNumApplications()
+ " applications,"
@ -646,7 +646,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for the user on this queue
User user = usersManager.getUserAndAddIfAbsent(userName);
//TODO recalculate max applications because they can depend on capacity
if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) {
if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
!(this instanceof AutoCreatedLeafQueue)) {
String msg = "Queue " + getQueuePath() + " already has " + user
.getTotalApplications() + " applications from user " + userName
+ " cannot accept submission of application: " + applicationId;
@ -825,10 +826,9 @@ public class AbstractLeafQueue extends AbstractCSQueue {
calculateAndGetAMResourceLimitPerPartition(nodePartition);
}
for (Iterator<FiCaSchedulerApp> fsApp =
getPendingAppsOrderingPolicy()
for (Iterator<FiCaSchedulerApp> fsApp = getPendingAppsOrderingPolicy()
.getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR);
fsApp.hasNext(); ) {
fsApp.hasNext();) {
FiCaSchedulerApp application = fsApp.next();
ApplicationId applicationId = application.getApplicationId();
@ -864,7 +864,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
+ " skipping enforcement to allow at least one application"
+ " to start");
} else{
application.updateAMContainerDiagnostics(SchedulerApplicationAttempt.AMState.INACTIVATED,
application.updateAMContainerDiagnostics(
SchedulerApplicationAttempt.AMState.INACTIVATED,
CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
LOG.debug("Not activating application {} as amIfStarted: {}"
+ " exceeds amLimit: {}", applicationId, amIfStarted, amLimit);
@ -1189,9 +1190,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
boolean needAssignToQueueCheck = true;
IteratorSelector sel = new IteratorSelector();
sel.setPartition(candidates.getPartition());
for (Iterator<FiCaSchedulerApp> assignmentIterator =
orderingPolicy.getAssignmentIterator(sel);
assignmentIterator.hasNext(); ) {
for (Iterator<FiCaSchedulerApp> assignmentIterator = orderingPolicy.getAssignmentIterator(sel);
assignmentIterator.hasNext();) {
FiCaSchedulerApp application = assignmentIterator.next();
ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
@ -1821,13 +1821,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
RMNodeLabelsManager.NO_LABEL)) {
TreeSet<RMContainer> rmContainers = null;
if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(
nodePartition))) {
rmContainers = new TreeSet<>();
ignorePartitionExclusivityRMContainers.put(nodePartition,
rmContainers);
}
TreeSet<RMContainer> rmContainers = ignorePartitionExclusivityRMContainers.computeIfAbsent(
nodePartition, k -> new TreeSet<>());
rmContainers.add(rmContainer);
}
@ -2195,8 +2190,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
* @return all ignored partition exclusivity RMContainers in the LeafQueue,
* this will be used by preemption policy.
*/
public Map<String, TreeSet<RMContainer>>
getIgnoreExclusivityRMContainers() {
public Map<String, TreeSet<RMContainer>> getIgnoreExclusivityRMContainers() {
Map<String, TreeSet<RMContainer>> clonedMap = new HashMap<>();
readLock.lock();

View File

@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractAutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueueConfig;

View File

@ -275,8 +275,9 @@ public class TestApplicationLimits {
Resource clusterResource =
Resources.createResource(100 * 16 * GB, 100 * 16);
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB, 1),
Resources.createResource(16*GB, 16), clusterResource);
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
Resources.createResource(GB, 1), Resources.createResource(16*GB, 16),
clusterResource);
CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
@ -299,7 +300,7 @@ public class TestApplicationLimits {
assertThat(queue.calculateAndGetAMResourceLimit()).
isEqualTo(amResourceLimit);
assertThat(queue.getUserAMResourceLimit()).isEqualTo(
Resource.newInstance(80*GB, 1));
Resource.newInstance(80*GB, 1));
// Assert in metrics
assertThat(queue.getMetrics().getAMResourceLimitMB()).isEqualTo(
@ -307,10 +308,8 @@ public class TestApplicationLimits {
assertThat(queue.getMetrics().getAMResourceLimitVCores()).isEqualTo(
amResourceLimit.getVirtualCores());
assertEquals(
(int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB());
// Add some nodes to the cluster & test new limits
clusterResource = Resources.createResource(120 * 16 * GB);
@ -322,10 +321,8 @@ public class TestApplicationLimits {
assertThat(queue.getUserAMResourceLimit()).isEqualTo(
Resource.newInstance(96*GB, 1));
assertEquals(
(int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB());
// should return -1 if per queue setting not set
assertEquals(
@ -343,11 +340,10 @@ public class TestApplicationLimits {
assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
// should default to global setting if per queue setting not set
assertEquals(
(long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
assertEquals((long)
CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
(long)csConf.getMaximumApplicationMasterResourcePerQueuePercent(
queue.getQueuePath())
);
queue.getQueuePath()));
// Change the per-queue max AM resources percentage.
csConf.setFloat(PREFIX + queue.getQueuePath()
@ -367,8 +363,7 @@ public class TestApplicationLimits {
assertEquals((long) 0.5,
(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
queue.getQueuePath())
);
queue.getQueuePath()));
assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo(
Resource.newInstance(800 * GB, 1));
@ -579,8 +574,8 @@ public class TestApplicationLimits {
// Say cluster has 100 nodes of 16G each
Resource clusterResource = Resources.createResource(100 * 16 * GB);
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB),
Resources.createResource(16*GB), clusterResource);
CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
Resources.createResource(GB), Resources.createResource(16*GB), clusterResource);
CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);

View File

@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;

View File

@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;