YARN-2689 TestSecureRMRegistryOperations failing on windows: secure ZK won't start

hadoop-yarn-project/CHANGES.txt

@@ -609,6 +609,11 @@ Release 2.6.0 - UNRELEASED
     YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
 
     YARN-2668 yarn-registry JAR won't link against ZK 3.4.5. (stevel)
+    
+    YARN-2689 TestSecureRMRegistryOperations failing on windows:
+    secure ZK won't start (stevel)
+
+    ---
 
     YARN-2598 GHS should show N/A instead of null for the inaccessible information
     (Zhijie Shen via mayank)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java

@@ -249,9 +249,6 @@ public class CuratorService extends CompositeService
     synchronized (CuratorService.class) {
       // set the security options
 
-      //log them
-      securityConnectionDiagnostics = buildSecurityDiagnostics();
-
       // build up the curator itself
       CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
       builder.ensembleProvider(ensembleProvider)
@@ -264,7 +261,8 @@ public class CuratorService extends CompositeService
 
       // set up the builder AND any JVM context
       registrySecurity.applySecurityEnvironment(builder);
-
+      //log them
+      securityConnectionDiagnostics = buildSecurityDiagnostics();
       framework = builder.build();
       framework.start();
     }
@@ -275,7 +273,7 @@ public class CuratorService extends CompositeService
   @Override
   public String toString() {
     return super.toString()
-           + bindingDiagnosticDetails();
+           + " " + bindingDiagnosticDetails();
   }
 
   /**
@@ -386,7 +384,9 @@ public class CuratorService extends CompositeService
       ioe = new PathIsNotEmptyDirectoryException(path);
     } else if (exception instanceof KeeperException.AuthFailedException) {
       ioe = new AuthenticationFailedException(path,
-          "Authentication Failed: " + exception, exception);
+          "Authentication Failed: " + exception
+          + "; " + securityConnectionDiagnostics,
+          exception);
     } else if (exception instanceof KeeperException.NoChildrenForEphemeralsException) {
       ioe = new NoChildrenForEphemeralsException(path,
           "Cannot create a path under an ephemeral node: " + exception,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java

@@ -596,6 +596,7 @@ public class RegistrySecurity extends AbstractService {
       + " %s required\n"
       // kerberos module
       + " keyTab=\"%s\"\n"
+      + " debug=true\n"
       + " principal=\"%s\"\n"
       + " useKeyTab=true\n"
       + " useTicketCache=false\n"
@@ -621,12 +622,15 @@ public class RegistrySecurity extends AbstractService {
         "invalid context");
     Preconditions.checkArgument(keytab != null && keytab.isFile(),
         "Keytab null or missing: ");
+    String keytabpath = keytab.getAbsolutePath();
+    // fix up for windows; no-op on unix
+    keytabpath =  keytabpath.replace('\\', '/');
     return String.format(
         Locale.ENGLISH,
         JAAS_ENTRY,
         context,
         getKerberosAuthModuleForJVM(),
-        keytab.getAbsolutePath(),
+        keytabpath,
         principal);
   }
 
@@ -846,11 +850,11 @@ public class RegistrySecurity extends AbstractService {
     StringBuilder builder = new StringBuilder();
     builder.append(secureRegistry ? "secure registry; "
                           : "insecure registry; ");
-    builder.append("Access policy: ").append(access);
+    builder.append("Curator service access policy: ").append(access);
 
-    builder.append(", System ACLs: ").append(aclsToString(systemACLs));
+    builder.append("; System ACLs: ").append(aclsToString(systemACLs));
-    builder.append(UgiInfo.fromCurrentUser());
+    builder.append("User: ").append(UgiInfo.fromCurrentUser());
-    builder.append(" Kerberos Realm: ").append(kerberosRealm).append(" ; ");
+    builder.append("; Kerberos Realm: ").append(kerberosRealm);
     builder.append(describeProperty(Environment.JAAS_CONF_KEY));
     String sasl =
         System.getProperty(PROP_ZK_ENABLE_SASL_CLIENT,
@@ -859,7 +863,7 @@ public class RegistrySecurity extends AbstractService {
     builder.append(describeProperty(PROP_ZK_ENABLE_SASL_CLIENT,
         DEFAULT_ZK_ENABLE_SASL_CLIENT));
     if (saslEnabled) {
-      builder.append("JAAS Client Identity")
+      builder.append("; JAAS Client Identity")
              .append("=")
              .append(jaasClientIdentity)
              .append("; ");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java

@@ -46,6 +46,7 @@ import javax.security.auth.kerberos.KerberosPrincipal;
 import javax.security.auth.login.LoginContext;
 import javax.security.auth.login.LoginException;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.security.Principal;
 import java.util.HashSet;
@@ -319,11 +320,16 @@ public class AbstractSecureRegistryTest extends RegistryTestHelper {
    * @param keytab keytab
    * @return the logged in context
    * @throws LoginException failure to log in
+   * @throws FileNotFoundException no keytab
    */
   protected LoginContext login(String principal,
-      String context, File keytab) throws LoginException {
+      String context, File keytab) throws LoginException,
+      FileNotFoundException {
     LOG.info("Logging in as {} in context {} with keytab {}",
         principal, context, keytab);
+    if (!keytab.exists()) {
+      throw new FileNotFoundException(keytab.getAbsolutePath());
+    }
     Set<Principal> principals = new HashSet<Principal>();
     principals.add(new KerberosPrincipal(principal));
     Subject subject = new Subject(false, principals, new HashSet<Object>(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java

@@ -99,12 +99,25 @@ public class TestSecureLogins extends AbstractSecureRegistryTest {
                                 ALICE_CLIENT_CONTEXT,
                                 keytab_alice);
 
-    logLoginDetails(ALICE_LOCALHOST, client);
+    try {
-    String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+      logLoginDetails(ALICE_LOCALHOST, client);
-    assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
+      String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
-    String config = FileUtils.readFileToString(new File(confFilename));
+      assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
-    LOG.info("{}=\n{}", confFilename, config);
+      String config = FileUtils.readFileToString(new File(confFilename));
-    RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+      LOG.info("{}=\n{}", confFilename, config);
+      RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+    } finally {
+      client.logout();
+    }
+  }
+
+  @Test
+  public void testZKServerContextLogin() throws Throwable {
+    LoginContext client = login(ZOOKEEPER_LOCALHOST,
+                                ZOOKEEPER_SERVER_CONTEXT,
+                                keytab_zk);
+    logLoginDetails(ZOOKEEPER_LOCALHOST, client);
+
     client.logout();
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java

@@ -24,12 +24,16 @@ import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
 import org.apache.hadoop.registry.client.impl.zk.CuratorService;
 import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
 import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Login;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+import org.apache.zookeeper.server.auth.SaslServerCallbackHandler;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.LoginContext;
 
 import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
@@ -52,6 +56,36 @@ public class TestSecureRegistry extends AbstractSecureRegistryTest {
     RegistrySecurity.clearZKSaslClientProperties();
   }
 
+  /**
+  * this is a cut and paste of some of the ZK internal code that was
+   * failing on windows and swallowing its exceptions
+   */
+  @Test
+  public void testLowlevelZKSaslLogin() throws Throwable {
+    RegistrySecurity.bindZKToServerJAASContext(ZOOKEEPER_SERVER_CONTEXT);
+    String serverSection =
+        System.getProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
+            ZooKeeperSaslServer.DEFAULT_LOGIN_CONTEXT_NAME);
+    assertEquals(ZOOKEEPER_SERVER_CONTEXT, serverSection);
+
+    AppConfigurationEntry entries[];
+    entries = javax.security.auth.login.Configuration.getConfiguration()
+                                                     .getAppConfigurationEntry(
+                                                         serverSection);
+
+    assertNotNull("null entries", entries);
+
+    SaslServerCallbackHandler saslServerCallbackHandler =
+        new SaslServerCallbackHandler(
+            javax.security.auth.login.Configuration.getConfiguration());
+    Login login = new Login(serverSection, saslServerCallbackHandler);
+    try {
+      login.startThreadIfNeeded();
+    } finally {
+      login.shutdown();
+    }
+  }
+
   @Test
   public void testCreateSecureZK() throws Throwable {
     startSecureZK();