Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir. Contributed by Adam Antal

This commit is contained in:
Szilard Nemeth 2020-02-25 21:30:04 +01:00
parent dda00d3ff5
commit d68616b045
3 changed files with 112 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
View File

@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,6 +40,13 @@ public class FSConfigToCSConfigArgumentHandler {
private static final Logger LOG =
LoggerFactory.getLogger(FSConfigToCSConfigArgumentHandler.class);
private static final String ALREADY_CONTAINS_EXCEPTION_MSG =
"The %s (provided with %s|%s arguments) contains " +
"the %s provided with the %s|%s options.";
private static final String ALREADY_CONTAINS_FILE_EXCEPTION_MSG =
"The %s %s (provided with %s|%s arguments) already contains a file " +
"or directory named %s which will be the output of the conversion!";
private FSConfigToCSConfigRuleHandler ruleHandler;
private FSConfigToCSConfigConverterParams converterParams;
private ConversionOptions conversionOptions;
@ -213,6 +221,7 @@ public class FSConfigToCSConfigArgumentHandler {
checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile);
checkFile(CliOption.CONVERSION_RULES, conversionRulesFile);
checkDirectory(CliOption.OUTPUT_DIR, outputDir);
checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
return FSConfigToCSConfigConverterParams.Builder.create()
.withYarnSiteXmlConfig(yarnSiteXmlFile)
@ -225,6 +234,45 @@ public class FSConfigToCSConfigArgumentHandler {
.build();
}
private static void checkOutputDirDoesNotContainXmls(String yarnSiteXmlFile,
String outputDir) {
if (yarnSiteXmlFile == null || outputDir == null) {
return;
}
// check whether yarn-site.xml is not in the output folder
File xmlFile = new File(yarnSiteXmlFile);
File xmlParentFolder = xmlFile.getParentFile();
File output = new File(outputDir);
if (output.equals(xmlParentFolder)) {
throw new IllegalArgumentException(
String.format(ALREADY_CONTAINS_EXCEPTION_MSG,
CliOption.OUTPUT_DIR.name, CliOption.OUTPUT_DIR.shortSwitch,
CliOption.OUTPUT_DIR.longSwitch, CliOption.YARN_SITE.name,
CliOption.YARN_SITE.shortSwitch,
CliOption.YARN_SITE.longSwitch));
}
// check whether the output folder does not contain nor yarn-site.xml
// neither capacity-scheduler.xml
checkFileNotInOutputDir(output,
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
checkFileNotInOutputDir(output,
YarnConfiguration.CS_CONFIGURATION_FILE);
}
private static void checkFileNotInOutputDir(File output, String fileName) {
File file = new File(output, fileName);
if (file.exists()) {
throw new IllegalArgumentException(
String.format(ALREADY_CONTAINS_FILE_EXCEPTION_MSG,
CliOption.OUTPUT_DIR.name, output,
CliOption.OUTPUT_DIR.shortSwitch,
CliOption.OUTPUT_DIR.longSwitch,
fileName));
}
}
private void printHelp(Options opts) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("General options are: ", opts);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
View File

@ -35,7 +35,7 @@ import static org.junit.Assert.assertTrue;
*
*/
public class FSConfigConverterTestCommons {
private final static String TEST_DIR =
public final static String TEST_DIR =
new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
public final static String FS_ALLOC_FILE =
new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath();

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
View File

@ -20,10 +20,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@ -492,4 +495,64 @@ public class TestFSConfigToCSConfigArgumentHandler {
assertFalse("No terminal rule check was enabled",
conversionOptions.isNoRuleTerminalCheck());
}
@Test
public void testYarnSiteOptionInOutputFolder() throws Exception {
setupFSConfigConversionFiles(true);
FSConfigToCSConfigArgumentHandler argumentHandler =
createArgumentHandler();
String[] args = new String[] {
"-y", FSConfigConverterTestCommons.YARN_SITE_XML,
"-o", FSConfigConverterTestCommons.TEST_DIR};
int retVal = argumentHandler.parseAndConvert(args);
assertEquals("Return value", -1, retVal);
assertTrue(fsTestCommons.getErrContent()
.toString().contains("contains the yarn-site.xml"));
}
private void testFileExistsInOutputFolder(String file) throws Exception {
File testFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, file);
try {
FileUtils.touch(testFile);
setupFSConfigConversionFiles(true);
FSConfigToCSConfigArgumentHandler argumentHandler =
createArgumentHandler();
String[] args = new String[] {
"-y", FSConfigConverterTestCommons.YARN_SITE_XML,
"-o", FSConfigConverterTestCommons.OUTPUT_DIR};
int retVal = argumentHandler.parseAndConvert(args);
assertEquals("Return value", -1, retVal);
String expectedMessage = String.format(
"already contains a file or directory named %s", file);
assertTrue(fsTestCommons.getErrContent()
.toString().contains(expectedMessage));
} finally {
if (testFile.exists()) {
testFile.delete();
}
}
}
@Test
public void testYarnSiteExistsInOutputFolder() throws Exception {
testFileExistsInOutputFolder(
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
}
@Test
public void testCapacitySchedulerXmlExistsInOutputFolder()
throws Exception {
testFileExistsInOutputFolder(
YarnConfiguration.CS_CONFIGURATION_FILE);
}
}