From 9262797e86453fc04b7ca3710b73b21fcdf9e6b4 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 27 Dec 2016 14:14:08 -0800 Subject: [PATCH] YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter) --- .../unmanagedamlauncher/UnmanagedAMLauncher.java | 10 +++++----- .../org/apache/hadoop/yarn/client/cli/TopCLI.java | 15 ++++++++------- .../yarn/client/api/impl/TimelineClientImpl.java | 2 +- .../java/org/apache/hadoop/yarn/state/Graph.java | 2 +- .../hadoop/yarn/state/VisualizeStateMachine.java | 4 +++- .../hadoop/yarn/util/ProcfsBasedProcessTree.java | 5 ++++- .../application/ApplicationImpl.java | 2 +- .../containermanager/container/ContainerImpl.java | 2 +- .../linux/resources/TrafficController.java | 13 +++++++------ .../localizer/LocalizedResource.java | 2 +- .../localizer/ResourceLocalizationService.java | 1 + 11 files changed, 33 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b9c7e802b7c..030a730884c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -192,11 +192,11 @@ public class UnmanagedAMLauncher { throw new RuntimeException(ex); } tokenFile.deleteOnExit(); - DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, - true)); - credentials.writeTokenStorageToStream(os); - os.close(); - + try (DataOutputStream os = new DataOutputStream( + new FileOutputStream(tokenFile, true))) { + credentials.writeTokenStorageToStream(os); + } + Map env = System.getenv(); ArrayList envAMList = new ArrayList(); boolean setClasspath = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index f99dd4885e7..cf7b7471859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -774,13 +774,14 @@ public class TopCLI extends YarnCLI { private JSONObject getJSONObject(URLConnection conn) throws IOException, JSONException { - InputStream in = conn.getInputStream(); - String encoding = conn.getContentEncoding(); - encoding = encoding == null ? "UTF-8" : encoding; - String body = IOUtils.toString(in, encoding); - JSONObject obj = new JSONObject(body); - JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); - return clusterInfo; + try(InputStream in = conn.getInputStream()) { + String encoding = conn.getContentEncoding(); + encoding = encoding == null ? "UTF-8" : encoding; + String body = IOUtils.toString(in, encoding); + JSONObject obj = new JSONObject(body); + JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); + return clusterInfo; + } } private URL getClusterUrl() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index b275a948546..4506c4868aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -805,7 +805,7 @@ public class TimelineClientImpl extends TimelineClient { error.getErrorCode()); } } - } else if (type.equals(DOMAIN_DATA_TYPE)) { + } else if (type.equals(DOMAIN_DATA_TYPE) && domains != null) { boolean hasError = false; for (TimelineDomain domain : domains.getDomains()) { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java index 390deb21bf5..6fb9119e21d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java @@ -189,7 +189,7 @@ public class Graph { public void save(String filepath) throws IOException { try (OutputStreamWriter fout = new OutputStreamWriter( - new FileOutputStream(filepath), Charset.forName("UTF-8"));) { + new FileOutputStream(filepath), Charset.forName("UTF-8"))) { fout.write(generateGraphViz()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java index 26e93c693e6..16203976e33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java @@ -49,7 +49,9 @@ public class VisualizeStateMachine { if (gname.endsWith("Impl")) { gname = gname.substring(0, gname.length()-4); } - ret.addSubGraph(factory.generateStateGraph(gname)); + if (ret != null) { + ret.addSubGraph(factory.generateStateGraph(gname)); + } } return ret; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 29bc277829b..bb2a77f0037 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -803,7 +803,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { if (LOG.isDebugEnabled()) { LOG.debug("MemInfo : " + key + " : Value : " + value); } - memoryMappingInfo.setMemInfo(key, value); + + if (memoryMappingInfo != null) { + memoryMappingInfo.setMemInfo(key, value); + } } } catch (Throwable t) { LOG diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index b9197c22943..112b43af878 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -603,7 +603,7 @@ public class ApplicationImpl implements Application { } catch (InvalidStateTransitionException e) { LOG.warn("Can't handle this event at current state", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { LOG.info("Application " + applicationID + " transitioned from " + oldState + " to " + newState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 4a6be32976f..cae30cde37d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1680,7 +1680,7 @@ public class ContainerImpl implements Container { + oldState + "], eventType: [" + event.getType() + "]," + " container: [" + containerID + "]", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { LOG.info("Container " + containerID + " transitioned from " + oldState + " to " + newState); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java index f1468fbac67..4698eb38c75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java @@ -626,15 +626,16 @@ import java.util.regex.Pattern; try { File tcCmds = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_SUFFIX, new File(tmpDirPath)); - Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds), - "UTF-8"); - PrintWriter printWriter = new PrintWriter(writer); - for (String command : commands) { - printWriter.println(command); + try ( + Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds), + "UTF-8"); + PrintWriter printWriter = new PrintWriter(writer)) { + for (String command : commands) { + printWriter.println(command); + } } - printWriter.close(); operation.appendArgs(tcCmds.getAbsolutePath()); return operation; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java index 6d67cac19c0..98f4e21d119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java @@ -200,7 +200,7 @@ public class LocalizedResource implements EventHandler { } catch (InvalidStateTransitionException e) { LOG.warn("Can't handle this event at current state", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { if (LOG.isDebugEnabled()) { LOG.debug("Resource " + resourcePath + (localPath != null ? "(->" + localPath + ")": "") + " transitioned from " + oldState diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index e4d74f65166..37473e3192c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -1051,6 +1051,7 @@ public class ResourceLocalizationService extends CompositeService LOG.error( "Got exception in parsing URL of LocalResource:" + rsrc.getResource(), e); + continue; } LocalizerResourceRequestEvent assoc = scheduled.get(req); if (assoc == null) {