diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2d83526e828..0f3d31e60dc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -63,6 +63,9 @@ Release 2.7.0 - UNRELEASED YARN-2857. ConcurrentModificationException in ContainerLogAppender (Mohammad Kamrul Islam via jlowe) + YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu + via jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 7cf4921157c..9d5468845da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -146,6 +146,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { throws IOException { ArrayList containers = new ArrayList(); + ArrayList containersToRemove = + new ArrayList(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); @@ -165,7 +167,14 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { ContainerId containerId = ConverterUtils.toContainerId( key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos)); String keyPrefix = key.substring(0, idEndPos+1); - containers.add(loadContainerState(containerId, iter, keyPrefix)); + RecoveredContainerState rcs = loadContainerState(containerId, + iter, keyPrefix); + // Don't load container without StartContainerRequest + if (rcs.startRequest != null) { + containers.add(rcs); + } else { + containersToRemove.add(containerId); + } } } catch (DBException e) { throw new IOException(e); @@ -175,6 +184,19 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } } + // remove container without StartContainerRequest + for (ContainerId containerId : containersToRemove) { + LOG.warn("Remove container " + containerId + + " with incomplete records"); + try { + removeContainer(containerId); + // TODO: kill and cleanup the leaked container + } catch (IOException e) { + LOG.error("Unable to remove container " + containerId + + " in store", e); + } + } + return containers; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 438cec3a793..f7f43cc5aa4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -274,6 +274,13 @@ public class TestNMLeveldbStateStoreService { assertEquals(containerReq, rcs.getStartRequest()); assertTrue(rcs.getDiagnostics().isEmpty()); + // store a new container record without StartContainerRequest + ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 6); + stateStore.storeContainerLaunched(containerId1); + recoveredContainers = stateStore.loadContainersState(); + // check whether the new container record is discarded + assertEquals(1, recoveredContainers.size()); + // launch the container, add some diagnostics, and verify recovered StringBuilder diags = new StringBuilder(); stateStore.storeContainerLaunched(containerId);