Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla 

(cherry picked from commit f5acf94eca)
This commit is contained in:
Jason Lowe 2015-11-24 23:47:10 +00:00
parent 9e54433c6c
commit 3a2f5d6329
3 changed files with 39 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/CHANGES.txt
View File

@ -1048,6 +1048,9 @@ Release 2.7.3 - UNRELEASED
YARN-4344. NMs reconnecting with changed capabilities can lead to wrong
cluster resource calculations (Varun Vasudev via jlowe)
YARN-4365. FileSystemNodeLabelStore should check for root dir existence on
startup (Kuhu Shukla via jlowe)
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@ -1914,6 +1917,9 @@ Release 2.6.3 - UNRELEASED
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
log files from full disks. (zhihai xu via jlowe)
YARN-4365. FileSystemNodeLabelStore should check for root dir existence on
startup (Kuhu Shukla via jlowe)
Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
View File

@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
setFileSystem(conf);
// mkdir of root dir path
fs.mkdirs(fsWorkingPath);
if (!fs.exists(fsWorkingPath)) {
fs.mkdirs(fsWorkingPath);
}
}
@Override
@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
IOUtils.cleanup(LOG, fs, editlogOs);
}
private void setFileSystem(Configuration conf) throws IOException {
void setFileSystem(Configuration conf) throws IOException {
Configuration confCopy = new Configuration(conf);
confCopy.setBoolean("dfs.client.retry.policy.enabled", true);
String retryPolicy =

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
View File

@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.InlineDispatcher;
@ -33,6 +35,7 @@ import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.ImmutableMap;
import org.mockito.Mockito;
public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
MockNodeLabelManager mgr = null;
@ -317,4 +320,30 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9")));
mgr.stop();
}
@Test
public void testRootMkdirOnInitStore() throws Exception {
final FileSystem mockFs = Mockito.mock(FileSystem.class);
FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) {
void setFileSystem(Configuration conf) throws IOException {
fs = mockFs;
}
};
mockStore.fs = mockFs;
verifyMkdirsCount(mockStore, true, 0);
verifyMkdirsCount(mockStore, false, 1);
verifyMkdirsCount(mockStore, true, 1);
verifyMkdirsCount(mockStore, false, 2);
}
private void verifyMkdirsCount(FileSystemNodeLabelsStore store,
boolean existsRetVal, int expectedNumOfCalls)
throws Exception {
Mockito.when(store.fs.exists(Mockito.any(
Path.class))).thenReturn(existsRetVal);
store.init(conf);
Mockito.verify(store.fs,Mockito.times(
expectedNumOfCalls)).mkdirs(Mockito.any(Path
.class));
}
}