Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6519. Fix warnings from Spotbugs in hadoop-yarn-server-resourcemanager. Contributed by Weiwei Yang.

This commit is contained in:
Naganarasimha 2017-05-01 20:15:27 +05:30
parent 64f68cb0b8
commit 30fc580196
11 changed files with 50 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
View File

@ -393,7 +393,7 @@ public boolean hasApplicationMasterRegistered(
return hasApplicationMasterRegistered;
}
protected final static List<Container> EMPTY_CONTAINER_LIST =
private final static List<Container> EMPTY_CONTAINER_LIST =
new ArrayList<Container>();
protected static final Allocation EMPTY_ALLOCATION = new Allocation(
EMPTY_CONTAINER_LIST, Resources.createResource(0), null, null, null);

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
View File

@ -52,7 +52,6 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@ -306,16 +305,12 @@ private void syncKillableContainersFromScheduler() {
private void cleanupStaledPreemptionCandidates(long currentTime) {
// Keep the preemptionCandidates list clean
for (Iterator<RMContainer> i = preemptionCandidates.keySet().iterator();
i.hasNext(); ) {
RMContainer id = i.next();
// garbage collect containers that are irrelevant for preemption
// And avoid preempt selected containers for *this execution*
// or within 1 ms
if (preemptionCandidates.get(id) + 2 * maxWaitTime < currentTime) {
i.remove();
}
}
// garbage collect containers that are irrelevant for preemption
// And avoid preempt selected containers for *this execution*
// or within 1 ms
preemptionCandidates.entrySet()
.removeIf(candidate ->
candidate.getValue() + 2 * maxWaitTime < currentTime);
}
private Set<String> getLeafQueueNames(TempQueuePerPartition q) {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -1000,9 +1000,11 @@ public void transferStateFromAttempt(RMAppAttempt attempt) {
// if am crashed and not received this response, we should resend
// this msg again after am restart
if (!this.finishedContainersSentToAM.isEmpty()) {
for (NodeId nodeId : this.finishedContainersSentToAM.keySet()) {
for (Map.Entry<NodeId, List<ContainerStatus>> finishedContainer
: this.finishedContainersSentToAM.entrySet()) {
List<ContainerStatus> containerStatuses =
this.finishedContainersSentToAM.get(nodeId);
finishedContainer.getValue();
NodeId nodeId = finishedContainer.getKey();
this.justFinishedContainers.putIfAbsent(nodeId, new ArrayList<>());
this.justFinishedContainers.get(nodeId).addAll(containerStatuses);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
View File

@ -152,7 +152,7 @@ public void updateAggregatePreemptedAppResourceUsage(
public void incNumAllocatedContainers(NodeType containerType,
NodeType requestType) {
localityStatistics[containerType.index][requestType.index]++;
localityStatistics[containerType.getIndex()][requestType.getIndex()]++;
totalAllocatedContainers++;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -132,7 +132,7 @@ public abstract class AbstractYarnScheduler
protected int nmExpireInterval;
protected long nmHeartbeatInterval;
protected final static List<Container> EMPTY_CONTAINER_LIST =
private final static List<Container> EMPTY_CONTAINER_LIST =
new ArrayList<Container>();
protected static final Allocation EMPTY_ALLOCATION = new Allocation(
EMPTY_CONTAINER_LIST, Resources.createResource(0), null, null, null);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeType.java
View File

@ -23,9 +23,17 @@
*/
public enum NodeType {
NODE_LOCAL(0), RACK_LOCAL(1), OFF_SWITCH(2);
public int index;
private NodeType(int index) {
private final int index;
NodeType(int index) {
this.index = index;
}
/**
* @return the index of the node type
*/
public int getIndex() {
return index;
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
View File

@ -142,20 +142,29 @@ static QueueMetrics forQueue(String queueName, Queue parent,
*/
@Private
public synchronized static void clearQueueMetrics() {
queueMetrics.clear();
QUEUE_METRICS.clear();
}
/**
* Simple metrics cache to help prevent re-registrations.
*/
protected final static Map<String, QueueMetrics> queueMetrics =
private static final Map<String, QueueMetrics> QUEUE_METRICS =
new HashMap<String, QueueMetrics>();
/**
* Returns the metrics cache to help prevent re-registrations.
*
* @return A string to {@link QueueMetrics} map.
*/
protected static Map<String, QueueMetrics> getQueueMetrics() {
return QUEUE_METRICS;
}
public synchronized
static QueueMetrics forQueue(MetricsSystem ms, String queueName,
Queue parent, boolean enableUserMetrics,
Configuration conf) {
QueueMetrics metrics = queueMetrics.get(queueName);
QueueMetrics metrics = QUEUE_METRICS.get(queueName);
if (metrics == null) {
metrics =
new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf).
@ -168,7 +177,7 @@ static QueueMetrics forQueue(MetricsSystem ms, String queueName,
sourceName(queueName).toString(),
"Metrics for queue: " + queueName, metrics);
}
queueMetrics.put(queueName, metrics);
QUEUE_METRICS.put(queueName, metrics);
}
return metrics;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
View File

@ -115,7 +115,7 @@ public void setAbsoluteUsedCapacity(Float absoluteUsedCapacity) {
public synchronized static CSQueueMetrics forQueue(String queueName,
Queue parent, boolean enableUserMetrics, Configuration conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
QueueMetrics metrics = queueMetrics.get(queueName);
QueueMetrics metrics = QueueMetrics.getQueueMetrics().get(queueName);
if (metrics == null) {
metrics =
new CSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
@ -127,7 +127,7 @@ public synchronized static CSQueueMetrics forQueue(String queueName,
ms.register(sourceName(queueName).toString(), "Metrics for queue: "
+ queueName, metrics);
}
queueMetrics.put(queueName, metrics);
QueueMetrics.getQueueMetrics().put(queueName, metrics);
}
return (CSQueueMetrics) metrics;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
View File

@ -69,9 +69,11 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
new Comparator<CSQueue>() {
@Override
public int compare(CSQueue q1, CSQueue q2) {
if (q1.getUsedCapacity() < q2.getUsedCapacity()) {
int result = Float.compare(q1.getUsedCapacity(),
q2.getUsedCapacity());
if (result < 0) {
return -1;
} else if (q1.getUsedCapacity() > q2.getUsedCapacity()) {
} else if (result > 0) {
return 1;
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
View File

@ -206,7 +206,7 @@ static FSQueueMetrics forQueue(String queueName, Queue parent,
public synchronized
static FSQueueMetrics forQueue(MetricsSystem ms, String queueName,
Queue parent, boolean enableUserMetrics, Configuration conf) {
QueueMetrics metrics = queueMetrics.get(queueName);
QueueMetrics metrics = QueueMetrics.getQueueMetrics().get(queueName);
if (metrics == null) {
metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
.tag(QUEUE_INFO, queueName);
@ -217,7 +217,7 @@ static FSQueueMetrics forQueue(MetricsSystem ms, String queueName,
sourceName(queueName).toString(),
"Metrics for queue: " + queueName, metrics);
}
queueMetrics.put(queueName, metrics);
QueueMetrics.getQueueMetrics().put(queueName, metrics);
}
return (FSQueueMetrics)metrics;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
View File

@ -155,10 +155,10 @@ synchronized LinkedHashMap<FSAppAttempt, Resource> getPreemptionList() {
* Remove apps that have their preemption requests fulfilled.
*/
private synchronized void cleanupPreemptionList() {
Iterator<FSAppAttempt> iterator =
resourcesPreemptedForApp.keySet().iterator();
while (iterator.hasNext()) {
FSAppAttempt app = iterator.next();
Iterator<Map.Entry<FSAppAttempt, Resource>> iterator =
resourcesPreemptedForApp.entrySet().iterator();
while(iterator.hasNext()) {
FSAppAttempt app = iterator.next().getKey();
if (app.isStopped() || !app.isStarved()) {
// App does not need more resources
Resources.subtractFrom(totalResourcesPreempted,