YARN-11308. Router Page display the db username and password in mask mode. (#4908)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java

@@ -98,7 +98,7 @@ public class ConfServlet extends HttpServlet {
     if (FORMAT_JSON.equals(format)) {
       Configuration.dumpConfiguration(conf, propertyName, out);
     } else if (FORMAT_XML.equals(format)) {
-      conf.writeXml(propertyName, out);
+      conf.writeXml(propertyName, out, conf);
     } else {
       throw new BadFormatException("Bad format: " + format);
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java

@@ -37,6 +37,7 @@ import org.apache.hadoop.util.StringUtils;
 public class ConfigRedactor {
 
   private static final String REDACTED_TEXT = "<redacted>";
+  private static final String REDACTED_XML = "******";
 
   private List<Pattern> compiledPatterns;
 
@@ -84,4 +85,19 @@ public class ConfigRedactor {
     }
     return false;
   }
+
+  /**
+   * Given a key / value pair, decides whether or not to redact and returns
+   * either the original value or text indicating it has been redacted.
+   *
+   * @param key param key.
+   * @param value param value, will return if conditions permit.
+   * @return Original value, or text indicating it has been redacted
+   */
+  public String redactXml(String key, String value) {
+    if (configIsSensitive(key)) {
+      return REDACTED_XML;
+    }
+    return value;
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -3593,11 +3593,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    * </ul>
    * @param propertyName xml property name.
    * @param out the writer to write to.
+   * @param config configuration.
    * @throws IOException raised on errors performing I/O.
    */
-  public void writeXml(@Nullable String propertyName, Writer out)
+  public void writeXml(@Nullable String propertyName, Writer out, Configuration config)
       throws IOException, IllegalArgumentException {
-    Document doc = asXmlDocument(propertyName);
+    ConfigRedactor redactor = config != null ? new ConfigRedactor(this) : null;
+    Document doc = asXmlDocument(propertyName, redactor);
 
     try {
       DOMSource source = new DOMSource(doc);
@@ -3614,11 +3616,16 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     }
   }
 
+  public void writeXml(@Nullable String propertyName, Writer out)
+      throws IOException, IllegalArgumentException {
+    writeXml(propertyName, out, null);
+  }
+
   /**
    * Return the XML DOM corresponding to this Configuration.
    */
-  private synchronized Document asXmlDocument(@Nullable String propertyName)
+  private synchronized Document asXmlDocument(@Nullable String propertyName,
-      throws IOException, IllegalArgumentException {
+      ConfigRedactor redactor) throws IOException, IllegalArgumentException {
     Document doc;
     try {
       doc = DocumentBuilderFactory
@@ -3641,13 +3648,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
             propertyName + " not found");
       } else {
         // given property is found, write single property
-        appendXMLProperty(doc, conf, propertyName);
+        appendXMLProperty(doc, conf, propertyName, redactor);
         conf.appendChild(doc.createTextNode("\n"));
       }
     } else {
       // append all elements
       for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
-        appendXMLProperty(doc, conf, (String)e.nextElement());
+        appendXMLProperty(doc, conf, (String)e.nextElement(), redactor);
         conf.appendChild(doc.createTextNode("\n"));
       }
     }
@@ -3663,7 +3670,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    * @param propertyName
    */
   private synchronized void appendXMLProperty(Document doc, Element conf,
-      String propertyName) {
+      String propertyName, ConfigRedactor redactor) {
     // skip writing if given property name is empty or null
     if (!Strings.isNullOrEmpty(propertyName)) {
       String value = properties.getProperty(propertyName);
@@ -3676,8 +3683,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         propNode.appendChild(nameNode);
 
         Element valueNode = doc.createElement("value");
-        valueNode.appendChild(doc.createTextNode(
+        String propertyValue = properties.getProperty(propertyName);
-            properties.getProperty(propertyName)));
+        if (redactor != null) {
+          propertyValue = redactor.redactXml(propertyName, propertyValue);
+        }
+        valueNode.appendChild(doc.createTextNode(propertyValue));
         propNode.appendChild(valueNode);
 
         Element finalNode = doc.createElement("final");

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -1000,6 +1000,7 @@ public class CommonConfigurationKeysPublic {
       String.join(",",
           "secret$",
           "password$",
+          "username$",
           "ssl.keystore.pass$",
           "fs.s3.*[Ss]ecret.?[Kk]ey",
           "fs.s3a.*.server-side-encryption.key",

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java

@@ -59,6 +59,7 @@ public class TestConfServlet {
       new HashMap<String, String>();
   private static final Map<String, String> TEST_FORMATS =
       new HashMap<String, String>();
+  private static final Map<String, String> MASK_PROPERTIES = new HashMap<>();
 
   @BeforeClass
   public static void initTestProperties() {
@@ -67,6 +68,8 @@ public class TestConfServlet {
     TEST_PROPERTIES.put("test.key3", "value3");
     TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml");
     TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json");
+    MASK_PROPERTIES.put("yarn.federation.state-store.sql.username", "admin");
+    MASK_PROPERTIES.put("yarn.federation.state-store.sql.password", "123456");
   }
 
   private Configuration getTestConf() {
@@ -80,6 +83,9 @@ public class TestConfServlet {
     for(String key : TEST_PROPERTIES.keySet()) {
       testConf.set(key, TEST_PROPERTIES.get(key));
     }
+    for(String key : MASK_PROPERTIES.keySet()) {
+      testConf.set(key, MASK_PROPERTIES.get(key));
+    }
     return testConf;
   }
 
@@ -247,4 +253,63 @@ public class TestConfServlet {
     }
     assertEquals("", sw.toString());
   }
+
+  private void verifyReplaceProperty(Configuration conf, String format,
+      String propertyName) throws Exception {
+    StringWriter sw = null;
+    PrintWriter pw = null;
+    ConfServlet service = null;
+    try {
+      service = new ConfServlet();
+      ServletConfig servletConf = mock(ServletConfig.class);
+      ServletContext context = mock(ServletContext.class);
+      service.init(servletConf);
+      when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
+      when(service.getServletContext()).thenReturn(context);
+
+      HttpServletRequest request = mock(HttpServletRequest.class);
+      when(request.getHeader(HttpHeaders.ACCEPT)).thenReturn(TEST_FORMATS.get(format));
+      when(request.getParameter("name")).thenReturn(propertyName);
+
+      HttpServletResponse response = mock(HttpServletResponse.class);
+      sw = new StringWriter();
+      pw = new PrintWriter(sw);
+      when(response.getWriter()).thenReturn(pw);
+
+      // response request
+      service.doGet(request, response);
+      String result = sw.toString().trim();
+
+      // For example, for the property yarn.federation.state-store.sql.username,
+      // we set the value to test-user,
+      // which should be replaced by a mask, which should be ******
+      // MASK_PROPERTIES.get("property yarn.federation.state-store.sql.username")
+      // is the value before replacement, test-user
+      // result contains the replaced value, which should be ******
+      assertTrue(result.contains(propertyName));
+      assertFalse(result.contains(MASK_PROPERTIES.get(propertyName)));
+
+    } finally {
+      if (sw != null) {
+        sw.close();
+      }
+      if (pw != null) {
+        pw.close();
+      }
+      if (service != null) {
+        service.destroy();
+      }
+    }
+  }
+
+  @Test
+  public void testReplaceProperty() throws Exception {
+    Configuration configurations = getMultiPropertiesConf();
+
+    for(String format : TEST_FORMATS.keySet()) {
+      for(String key : MASK_PROPERTIES.keySet()) {
+        verifyReplaceProperty(configurations, format, key);
+      }
+    }
+  }
 }