YARN-9899. Migration tool that help to generate CS config based on FS config [Phase 2]. Contributed by Peter Bacsko

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -47,6 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "queue" client "prints queue information"
   hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
   hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
+  hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)"
   hadoop_add_subcommand "rmadmin" admin "admin tools"
   hadoop_add_subcommand "router" daemon "run the Router daemon"
   hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
@@ -165,6 +166,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
       hadoop_translate_cygwin_path sld
       hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
     ;;
+    fs2cs)
+      HADOOP_CLASSNAME="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterMain"
+    ;;
     rmadmin)
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
     ;;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -21,10 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
 import com.google.common.annotations.VisibleForTesting;
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler.CliOption;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -232,13 +228,6 @@ public class ResourceManager extends CompositeService
   private Configuration conf;
 
   private UserGroupInformation rmLoginUGI;
-  private static FSConfigToCSConfigArgumentHandler
-      fsConfigConversionArgumentHandler;
-
-  static {
-    FSConfigToCSConfigConverter converter = initFSConfigConverter();
-    initFSArgumentHandler(converter);
-  }
 
   public ResourceManager() {
     super("ResourceManager");
@@ -1576,22 +1565,6 @@ public class ResourceManager extends CompositeService
         } else if (argv[0].equals("-remove-application-from-state-store")
             && argv.length == 2) {
           removeApplication(conf, argv[1]);
-        } else if (argv[0].equals("-convert-fs-configuration")) {
-          String[] args = Arrays.copyOfRange(argv, 1, argv.length);
-          try {
-            int exitCode =
-                fsConfigConversionArgumentHandler.parseAndConvert(args);
-            if (exitCode != 0) {
-              LOG.error(FATAL,
-                  "Error while starting FS configuration conversion, " +
-                      "see previous error messages for details!");
-              System.exit(exitCode);
-            }
-          } catch (Throwable t) {
-            LOG.error(FATAL,
-                "Error while starting FS configuration conversion!", t);
-            System.exit(-1);
-          }
         } else {
           printUsage(System.err);
         }
@@ -1744,12 +1717,6 @@ public class ResourceManager extends CompositeService
     out.println("                            "
         + "[-format-conf-store]" + "\n");
 
-    out.println("[-convert-fs-configuration ");
-    out.println(FSConfigToCSConfigConverter.WARNING_TEXT);
-    for (CliOption cliOption : CliOption.values()) {
-      out.println("   " + cliOption.getAsArgumentString());
-    }
-    out.println("]");
   }
 
   protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() {
@@ -1767,17 +1734,4 @@ public class ResourceManager extends CompositeService
   public boolean isSecurityEnabled() {
     return UserGroupInformation.isSecurityEnabled();
   }
-
-  @VisibleForTesting
-  static void initFSArgumentHandler(FSConfigToCSConfigConverter converter) {
-    ResourceManager.fsConfigConversionArgumentHandler =
-        new FSConfigToCSConfigArgumentHandler(converter);
-  }
-
-  private static FSConfigToCSConfigConverter initFSConfigConverter() {
-    FSConfigToCSConfigRuleHandler ruleHandler =
-        new FSConfigToCSConfigRuleHandler();
-    return new FSConfigToCSConfigConverter(ruleHandler);
-  }
-
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java

@@ -18,9 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,69 +48,76 @@ public class FSConfigToCSConfigArgumentHandler {
    */
   public enum CliOption {
     YARN_SITE("yarn-site.xml", "y", "yarnsiteconfig",
-        "Path to a valid yarn-site.xml config file", true, true),
+        "Path to a valid yarn-site.xml config file", true),
 
     // fair-scheduler.xml is not mandatory
     // if FairSchedulerConfiguration.ALLOCATION_FILE is defined in yarn-site.xml
     FAIR_SCHEDULER("fair-scheduler.xml", "f", "fsconfig",
-        "Path to a valid fair-scheduler.xml config file", false, true),
+        "Path to a valid fair-scheduler.xml config file", true),
     CONVERSION_RULES("conversion rules config file", "r", "rulesconfig",
         "Optional parameter. If given, should specify a valid path to the " +
-            "conversion rules file (property format).", false, true),
+            "conversion rules file (property format).", true),
     CONSOLE_MODE("console mode", "p", "print",
         "If defined, the converted configuration will " +
-            "only be emitted to the console.", false, false),
+            "only be emitted to the console.", false),
     CLUSTER_RESOURCE("cluster resource", "c", "cluster-resource",
         "Needs to be given if maxResources is defined as percentages " +
             "for any queue, otherwise this parameter can be omitted.",
-            false, true),
+              true),
     OUTPUT_DIR("output directory", "o", "output-directory",
         "Output directory for yarn-site.xml and" +
             " capacity-scheduler.xml files." +
             "Must have write permission for user who is running this script.",
-            true, true);
+            true),
+    HELP("help", "h", "help", "Displays the list of options", false);
 
     private final String name;
     private final String shortSwitch;
     private final String longSwitch;
     private final String description;
-    private final boolean required;
     private final boolean hasArg;
 
     CliOption(String name, String shortSwitch, String longSwitch,
-        String description, boolean required, boolean hasArg) {
+        String description, boolean hasArg) {
       this.name = name;
       this.shortSwitch = shortSwitch;
       this.longSwitch = longSwitch;
       this.description = description;
-      this.required = required;
       this.hasArg = hasArg;
     }
 
     public Option createCommonsCliOption() {
       Option option = new Option(shortSwitch, longSwitch, hasArg, description);
-      option.setRequired(required);
       return option;
     }
-
-    public String getAsArgumentString() {
-      return shortSwitch + "|" + longSwitch + ": " + description;
-    }
   }
 
-  public int parseAndConvert(String[] args) throws Exception {
+  int parseAndConvert(String[] args) throws Exception {
     Options opts = createOptions();
 
     try {
+      if (args.length == 0) {
+        LOG.info("Missing command line arguments");
+        printHelp(opts);
+        return 0;
+      }
+
       CommandLine cliParser = new GnuParser().parse(opts, args);
+
+      if (cliParser.hasOption(CliOption.HELP.shortSwitch)) {
+        printHelp(opts);
+        return 0;
+      }
+
       checkOptionPresent(cliParser, CliOption.YARN_SITE);
-      checkOptionPresent(cliParser, CliOption.OUTPUT_DIR);
+      checkOutputDefined(cliParser);
 
       FSConfigToCSConfigConverterParams params = validateInputFiles(cliParser);
       converter.convert(params);
-    } catch (MissingArgumentException e) {
+    } catch (ParseException e) {
-      String msg = "Missing argument for options" + e.getMessage();
+      String msg = "Options parsing failed: " + e.getMessage();
       logAndStdErr(e, msg);
+      printHelp(opts);
       return -1;
     } catch (PreconditionException e) {
       String msg = "Cannot start FS config conversion due to the following"
@@ -131,7 +139,8 @@ public class FSConfigToCSConfigArgumentHandler {
   }
 
   private void logAndStdErr(Exception e, String msg) {
-    LOG.error(msg, e);
+    LOG.debug("Stack trace", e);
+    LOG.error(msg);
     System.err.println(msg);
   }
 
@@ -172,6 +181,11 @@ public class FSConfigToCSConfigArgumentHandler {
         .build();
   }
 
+  private void printHelp(Options opts) {
+    HelpFormatter formatter = new HelpFormatter();
+    formatter.printHelp("General options are: ", opts);
+  }
+
   private static void checkOptionPresent(CommandLine cliParser,
       CliOption cliOption) {
     if (!cliParser.hasOption(cliOption.shortSwitch)) {
@@ -181,6 +195,20 @@ public class FSConfigToCSConfigArgumentHandler {
     }
   }
 
+  private static void checkOutputDefined(CommandLine cliParser) {
+    boolean hasOutputDir =
+        cliParser.hasOption(CliOption.OUTPUT_DIR.shortSwitch);
+
+    boolean console =
+        cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch);
+
+    if (!console && !hasOutputDir) {
+      throw new PreconditionException(
+         "Output directory or console mode was not defined. Please" +
+          " use -h or --help to see command line switches");
+    }
+  }
+
   private static void checkFile(CliOption cliOption, String filePath) {
     checkFileInternal(cliOption, filePath, true);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+/**
+ * Main class that invokes the FS->CS converter.
+ *
+ */
+@SuppressWarnings("checkstyle:hideutilityclassconstructor")
+public class FSConfigToCSConfigConverterMain {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(FSConfigToCSConfigConverterMain.class);
+  private static final Marker FATAL =
+      MarkerFactory.getMarker("FATAL");
+
+  public static void main(String[] args) {
+    try {
+      FSConfigToCSConfigRuleHandler ruleHandler =
+          new FSConfigToCSConfigRuleHandler();
+      FSConfigToCSConfigConverter converter =
+          new FSConfigToCSConfigConverter(ruleHandler);
+      FSConfigToCSConfigArgumentHandler fsConfigConversionArgumentHandler =
+          new FSConfigToCSConfigArgumentHandler(converter);
+      int exitCode =
+          fsConfigConversionArgumentHandler.parseAndConvert(args);
+      if (exitCode != 0) {
+        LOG.error(FATAL,
+            "Error while starting FS configuration conversion, " +
+                "see previous error messages for details!");
+        System.exit(exitCode);
+      }
+    } catch (Throwable t) {
+      LOG.error(FATAL,
+          "Error while starting FS configuration conversion!", t);
+      System.exit(-1);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
@@ -51,15 +52,15 @@ class QueuePlacementConverter {
       ruleCount++;
       if (rule instanceof UserPlacementRule) {
         UserPlacementRule userRule = (UserPlacementRule) rule;
-        if (mapping.length() > 0) {
-          mapping.append(";");
-        }
 
         // nested rule
         if (userRule.getParentRule() != null) {
           handleNestedRule(mapping, userRule);
         } else {
           if (!userAsDefaultQueue) {
+            if (mapping.length() > 0) {
+              mapping.append(";");
+            }
             mapping.append("u:" + USER + ":" + USER);
           }
         }
@@ -82,8 +83,11 @@ class QueuePlacementConverter {
         mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName);
       } else if (rule instanceof SecondaryGroupExistingPlacementRule) {
         // TODO: wait for YARN-9840
+        if (mapping.length() > 0) {
+          mapping.append(";");
+        }
         mapping.append("u:" + USER + ":" + SECONDARY_GROUP);
-      } else {
+      } else if (!(rule instanceof RejectPlacementRule)) {
         throw new IllegalArgumentException("Unknown placement rule: " + rule);
       }
     }
@@ -99,6 +103,9 @@ class QueuePlacementConverter {
   private void handleNestedRule(StringBuilder mapping,
       UserPlacementRule userRule) {
     PlacementRule pr = userRule.getParentRule();
+    if (mapping.length() > 0) {
+      mapping.append(";");
+    }
     if (pr instanceof PrimaryGroupPlacementRule) {
       // TODO: wait for YARN-9841
       mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java

@@ -18,6 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.TimeoutException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.http.lib.StaticUserWebFilter;
@@ -39,8 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterParams;
 import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
@@ -49,23 +54,8 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-import org.mockito.ArgumentCaptor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
 
 public class TestResourceManager {
   private static final Logger LOG =
@@ -372,71 +362,4 @@ public class TestResourceManager {
       dummyResourceManager.stop();
     }
   }
-
-  /**
-   * Example command: <br>
-   *   opt/hadoop/bin/yarn resourcemanager -convert-fs-configuration<br>
-   *   -o /tmp/output<br>
-   *   -y /opt/hadoop/etc/hadoop/yarn-site.xml<br>
-   *   -f /opt/hadoop/etc/hadoop/fair-scheduler.xml<br>
-   *   -r /home/systest/sample-rules-config.properties<br>
-   */
-  @Test
-  @SuppressWarnings("checkstyle:javadocstyle")
-  public void testResourceManagerConvertFSConfigurationDefaults()
-      throws Exception {
-    setupFSConfigConversionFiles();
-
-    ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
-        ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
-
-    final String mainSwitch = "-convert-fs-configuration";
-    FSConfigToCSConfigConverter mockConverter =
-        mock(FSConfigToCSConfigConverter.class);
-
-    ResourceManager.initFSArgumentHandler(mockConverter);
-    ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
-        "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
-        CONVERSION_RULES_FILE});
-
-    // validate params
-    verify(mockConverter).convert(conversionParams.capture());
-    FSConfigToCSConfigConverterParams params = conversionParams.getValue();
-    LOG.info("FS config converter parameters: " + params);
-
-    assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
-    assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
-    assertThat(params.getConversionRulesConfig())
-      .isEqualTo(CONVERSION_RULES_FILE);
-    assertThat(params.isConsole()).isEqualTo(false);
-  }
-
-  @Test
-  public void testResourceManagerConvertFSConfigurationWithConsoleParam()
-      throws Exception {
-    setupFSConfigConversionFiles();
-
-    ArgumentCaptor<FSConfigToCSConfigConverterParams> conversionParams =
-        ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
-
-    final String mainSwitch = "-convert-fs-configuration";
-    FSConfigToCSConfigConverter mockConverter =
-        mock(FSConfigToCSConfigConverter.class);
-
-    ResourceManager.initFSArgumentHandler(mockConverter);
-    ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR,
-        "-p", "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r",
-        CONVERSION_RULES_FILE});
-
-    // validate params
-    verify(mockConverter).convert(conversionParams.capture());
-    FSConfigToCSConfigConverterParams params = conversionParams.getValue();
-    LOG.info("FS config converter parameters: " + params);
-
-    assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML);
-    assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE);
-    assertThat(params.getConversionRulesConfig())
-      .isEqualTo(CONVERSION_RULES_FILE);
-    assertThat(params.isConsole()).isEqualTo(true);
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java

@@ -88,7 +88,11 @@ public class FSConfigConverterTestCommons {
     return errContent;
   }
 
-  private void deleteTestFiles() {
+  ByteArrayOutputStream getStdOutContent() {
+    return outContent;
+  }
+
+  public void deleteTestFiles() {
     //Files may not be created so we are not strict here!
     deleteFile(FS_ALLOC_FILE, false);
     deleteFile(YARN_SITE_XML, false);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java

@@ -17,7 +17,6 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
 
 import com.google.common.collect.Lists;
-import org.apache.commons.cli.MissingOptionException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -116,13 +115,13 @@ public class TestFSConfigToCSConfigArgumentHandler {
     String[] args = new String[] {"-o",
         FSConfigConverterTestCommons.OUTPUT_DIR};
 
-    expectedException.expect(MissingOptionException.class);
+    int retVal = argumentHandler.parseAndConvert(args);
-    expectedException.expectMessage("Missing required option: y");
+    assertEquals("Return value", -1, retVal);
 
-    argumentHandler.parseAndConvert(args);
+    assertTrue("Error content missing", fsTestCommons.getErrContent()
+        .toString().contains("Missing yarn-site.xml parameter"));
   }
 
-
   @Test
   public void testMissingFairSchedulerXmlArgument() throws Exception {
     setupFSConfigConversionFiles(true);
@@ -142,10 +141,12 @@ public class TestFSConfigToCSConfigArgumentHandler {
     String[] args = new String[] {"-y",
         FSConfigConverterTestCommons.YARN_SITE_XML};
 
-    expectedException.expect(MissingOptionException.class);
+    int retVal = argumentHandler.parseAndConvert(args);
-    expectedException.expectMessage("Missing required option: o");
+    assertEquals("Return value", -1, retVal);
 
-    argumentHandler.parseAndConvert(args);
+    assertTrue("Error content missing", fsTestCommons.getErrContent()
+        .toString()
+        .contains("Output directory or console mode was not defined"));
   }
 
   @Test
@@ -183,8 +184,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
         FSConfigConverterTestCommons.YARN_SITE_XML, "-o",
         FSConfigConverterTestCommons.YARN_SITE_XML);
 
-    argumentHandler.parseAndConvert(args);
+    int retVal = argumentHandler.parseAndConvert(args);
-    System.out.println(fsTestCommons.getErrContent());
+    assertEquals("Return value", -1, retVal);
     assertTrue("Error content missing", fsTestCommons.getErrContent()
         .toString()
         .contains("Cannot start FS config conversion due to the following " +
@@ -355,7 +356,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
     Mockito.doThrow(UnsupportedPropertyException.class)
       .when(mockConverter)
       .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
-    argumentHandler.parseAndConvert(args);
+    int retVal = argumentHandler.parseAndConvert(args);
+    assertEquals("Return value", -1, retVal);
     assertTrue("Error content missing", fsTestCommons.getErrContent()
         .toString().contains("Unsupported property/setting encountered"));
   }
@@ -372,7 +374,8 @@ public class TestFSConfigToCSConfigArgumentHandler {
 
     Mockito.doThrow(ConversionException.class).when(mockConverter)
       .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class));
-    argumentHandler.parseAndConvert(args);
+    int retVal = argumentHandler.parseAndConvert(args);
+    assertEquals("Return value", -1, retVal);
     assertTrue("Error content missing", fsTestCommons.getErrContent()
         .toString().contains("Fatal error during FS config conversion"));
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Unit tests for TestFSConfigToCSConfigConverterMain.
+ *
+ */
+public class TestFSConfigToCSConfigConverterMain {
+  private FSConfigConverterTestCommons converterTestCommons;
+
+  @Before
+  public void setUp() throws Exception {
+    converterTestCommons = new FSConfigConverterTestCommons();
+    converterTestCommons.setUp();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    converterTestCommons.tearDown();
+  }
+
+  /*
+   * Example command:
+   *   opt/hadoop/bin/yarn fs2cs
+   *   -o /tmp/output
+   *   -y /opt/hadoop/etc/hadoop/yarn-site.xml
+   *   -f /opt/hadoop/etc/hadoop/fair-scheduler.xml
+   *   -r /home/systest/sample-rules-config.properties
+   */
+  @Test
+  public void testConvertFSConfigurationDefaults()
+      throws Exception {
+    setupFSConfigConversionFiles();
+
+    FSConfigToCSConfigConverterMain.main(new String[] {
+        "-o", OUTPUT_DIR,
+        "-y", YARN_SITE_XML,
+        "-f", FS_ALLOC_FILE,
+        "-r", CONVERSION_RULES_FILE});
+
+    boolean csConfigExists =
+        new File(OUTPUT_DIR, "capacity-scheduler.xml").exists();
+    boolean yarnSiteConfigExists =
+        new File(OUTPUT_DIR, "yarn-site.xml").exists();
+
+    assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
+    assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
+  }
+
+  @Test
+  public void testConvertFSConfigurationWithConsoleParam()
+      throws Exception {
+    setupFSConfigConversionFiles();
+
+    FSConfigToCSConfigConverterMain.main(new String[] {
+        "-p",
+        "-y", YARN_SITE_XML,
+        "-f", FS_ALLOC_FILE,
+        "-r", CONVERSION_RULES_FILE});
+
+    String stdout = converterTestCommons.getStdOutContent().toString();
+    assertTrue("Stdout doesn't contain yarn-site.xml",
+        stdout.contains("======= yarn-site.xml ======="));
+    assertTrue("Stdout doesn't contain capacity-scheduler.xml",
+        stdout.contains("======= capacity-scheduler.xml ======="));
+  }
+
+  @Test
+  public void testShortHelpSwitch() {
+    FSConfigToCSConfigConverterMain.main(new String[] {"-h"});
+
+    verifyHelpText();
+  }
+
+  @Test
+  public void testLongHelpSwitch() {
+    FSConfigToCSConfigConverterMain.main(new String[] {"--help"});
+
+    verifyHelpText();
+  }
+
+  @Test
+  public void testConvertFSConfigurationWithLongSwitches()
+      throws IOException {
+    setupFSConfigConversionFiles();
+
+    FSConfigToCSConfigConverterMain.main(new String[] {
+        "--print",
+        "--yarnsiteconfig", YARN_SITE_XML,
+        "--fsconfig", FS_ALLOC_FILE,
+        "--rulesconfig", CONVERSION_RULES_FILE});
+
+    String stdout = converterTestCommons.getStdOutContent().toString();
+    assertTrue("Stdout doesn't contain yarn-site.xml",
+        stdout.contains("======= yarn-site.xml ======="));
+    assertTrue("Stdout doesn't contain capacity-scheduler.xml",
+        stdout.contains("======= capacity-scheduler.xml ======="));
+  }
+
+  private void verifyHelpText() {
+    String stdout = converterTestCommons.getStdOutContent().toString();
+    assertTrue("Help was not displayed",
+        stdout.contains("General options are:"));
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java

@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Unit tests for QueuePlacementConverter.
+ *
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class TestQueuePlacementConverter {
+  @Mock
+  private PlacementManager placementManager;
+
+  @Mock
+  private FSConfigToCSConfigRuleHandler ruleHandler;
+
+  private QueuePlacementConverter converter;
+
+  @Before
+  public void setup() {
+    this.converter = new QueuePlacementConverter();
+  }
+
+  @Test
+  public void testConvertUserAsDefaultQueue() {
+    Map<String, String> properties = convert(true);
+
+    verifyMapping(properties, "u:%user:%user");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertUserPlacementRuleWithoutUserAsDefaultQueue() {
+    testConvertUserPlacementRule(false);
+  }
+
+  @Test
+  public void testConvertUserPlacementRuleWithUserAsDefaultQueue() {
+    testConvertUserPlacementRule(true);
+  }
+
+  private void testConvertUserPlacementRule(boolean userAsDefaultQueue) {
+    PlacementRule rule = mock(UserPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(userAsDefaultQueue);
+
+    verifyMapping(properties, "u:%user:%user");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertSpecifiedPlacementRule() {
+    PlacementRule rule = mock(SpecifiedPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMappingNoOverride(properties, 1);
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertSpecifiedPlacementRuleAtSecondPlace() {
+    PlacementRule rule = mock(UserPlacementRule.class);
+    PlacementRule rule2 = mock(SpecifiedPlacementRule.class);
+    initPlacementManagerMock(rule, rule2);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMappingNoOverride(properties, 2);
+    verify(ruleHandler).handleSpecifiedNotFirstRule();
+  }
+
+  @Test
+  public void testConvertPrimaryGroupPlacementRule() {
+    PlacementRule rule = mock(PrimaryGroupPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:%primary_group");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertSecondaryGroupPlacementRule() {
+    PlacementRule rule = mock(SecondaryGroupExistingPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:%secondary_group");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertDefaultPlacementRule() {
+    DefaultPlacementRule rule = mock(DefaultPlacementRule.class);
+    rule.defaultQueueName = "abc";
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:abc");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testConvertUnsupportedPlacementRule() {
+    PlacementRule rule = mock(TestPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    // throws exception
+    convert(false);
+  }
+
+  @Test
+  public void testConvertRejectPlacementRule() {
+    PlacementRule rule = mock(RejectPlacementRule.class);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    assertEquals("Map is not empty", 0, properties.size());
+  }
+
+  @Test
+  public void testConvertNestedPrimaryGroupRule() {
+    UserPlacementRule rule = mock(UserPlacementRule.class);
+    PrimaryGroupPlacementRule parent = mock(PrimaryGroupPlacementRule.class);
+    when(rule.getParentRule()).thenReturn(parent);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:%primary_group.%user");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertNestedSecondaryGroupRule() {
+    UserPlacementRule rule = mock(UserPlacementRule.class);
+    SecondaryGroupExistingPlacementRule parent =
+        mock(SecondaryGroupExistingPlacementRule.class);
+    when(rule.getParentRule()).thenReturn(parent);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:%secondary_group.%user");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertNestedDefaultRule() {
+    UserPlacementRule rule = mock(UserPlacementRule.class);
+    DefaultPlacementRule parent =
+        mock(DefaultPlacementRule.class);
+    parent.defaultQueueName = "abc";
+    when(rule.getParentRule()).thenReturn(parent);
+    initPlacementManagerMock(rule);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties, "u:%user:abc.%user");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  @Test
+  public void testConvertMultiplePlacementRules() {
+    UserPlacementRule rule1 = mock(UserPlacementRule.class);
+    PrimaryGroupPlacementRule rule2 =
+        mock(PrimaryGroupPlacementRule.class);
+    SecondaryGroupExistingPlacementRule rule3 =
+        mock(SecondaryGroupExistingPlacementRule.class);
+    initPlacementManagerMock(rule1, rule2, rule3);
+
+    Map<String, String> properties = convert(false);
+
+    verifyMapping(properties,
+        "u:%user:%user;u:%user:%primary_group;u:%user:%secondary_group");
+    verifyZeroInteractions(ruleHandler);
+  }
+
+  private void initPlacementManagerMock(
+      PlacementRule... rules) {
+    List<PlacementRule> listOfRules = Lists.newArrayList(rules);
+    when(placementManager.getPlacementRules()).thenReturn(listOfRules);
+  }
+
+  private Map<String, String> convert(boolean userAsDefaultQueue) {
+    return converter.convertPlacementPolicy(placementManager, ruleHandler,
+        userAsDefaultQueue);
+  }
+
+  private void verifyMapping(Map<String, String> properties,
+      String expectedValue) {
+    assertEquals("Map size", 1, properties.size());
+    String value = properties.get(QUEUE_MAPPING);
+    assertNotNull("No mapping property found", value);
+    assertEquals("Mapping", expectedValue, value);
+  }
+
+  private void verifyMappingNoOverride(Map<String, String> properties,
+      int expectedSize) {
+    assertEquals("Map size", expectedSize, properties.size());
+    String value = properties.get(ENABLE_QUEUE_MAPPING_OVERRIDE);
+    assertNotNull("No mapping property found", value);
+    assertEquals("Override mapping", "false", value);
+  }
+
+  private class TestPlacementRule extends FSPlacementRule {
+    @Override
+    public ApplicationPlacementContext getPlacementForApp(
+        ApplicationSubmissionContext asc, String user) throws YarnException {
+      return null;
+    }
+  }
+}