HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster. (yzhangal via rkanter)

(cherry picked from commit 875256834b)

Conflicts:
	hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationToken.java
	hadoop-common-project/hadoop-common/CHANGES.txt

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -14,6 +14,7 @@
 package org.apache.hadoop.security.authentication.client;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.security.authentication.util.AuthToken;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.ietf.jgss.GSSContext;
 import org.ietf.jgss.GSSManager;
@@ -29,6 +30,7 @@ import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.Configuration;
 import javax.security.auth.login.LoginContext;
 import javax.security.auth.login.LoginException;
+
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -187,13 +189,18 @@ public class KerberosAuthenticator implements Authenticator {
       conn.setRequestMethod(AUTH_HTTP_METHOD);
       conn.connect();
       
+      boolean needFallback = false;
       if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
         LOG.debug("JDK performed authentication on our behalf.");
         // If the JDK already did the SPNEGO back-and-forth for
         // us, just pull out the token.
         AuthenticatedURL.extractToken(conn, token);
+        if (isTokenKerberos(token)) {
           return;
-      } else if (isNegotiate()) {
+        }
+        needFallback = true;
+      }
+      if (!needFallback && isNegotiate()) {
         LOG.debug("Performing our own SPNEGO sequence.");
         doSpnegoSequence(token);
       } else {
@@ -224,6 +231,21 @@ public class KerberosAuthenticator implements Authenticator {
     return auth;
   }
 
+  /*
+   * Check if the passed token is of type "kerberos" or "kerberos-dt"
+   */
+  private boolean isTokenKerberos(AuthenticatedURL.Token token)
+      throws AuthenticationException {
+    if (token.isSet()) {
+      AuthToken aToken = AuthToken.parse(token.toString());          
+      if (aToken.getType().equals("kerberos") ||
+          aToken.getType().equals("kerberos-dt")) {              
+        return true;
+      }
+    }
+    return false;
+  }
+
   /*
   * Indicates if the response is starting a SPNEGO negotiation.
   */

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java

@@ -14,14 +14,9 @@
 package org.apache.hadoop.security.authentication.server;
 
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.AuthToken;
 
 import java.security.Principal;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.StringTokenizer;
 
 import javax.servlet.http.HttpServletRequest;
 
@@ -34,38 +29,21 @@ import javax.servlet.http.HttpServletRequest;
  * and received in HTTP client responses and requests as a HTTP cookie (this is
  * done by the {@link AuthenticationFilter}).
  */
-public class AuthenticationToken implements Principal {
+public class AuthenticationToken extends AuthToken {
 
   /**
    * Constant that identifies an anonymous request.
    */
   public static final AuthenticationToken ANONYMOUS = new AuthenticationToken();
 
-  private static final String ATTR_SEPARATOR = "&";
-  private static final String USER_NAME = "u";
-  private static final String PRINCIPAL = "p";
-  private static final String EXPIRES = "e";
-  private static final String TYPE = "t";
-
-  private final static Set<String> ATTRIBUTES =
-    new HashSet<String>(Arrays.asList(USER_NAME, PRINCIPAL, EXPIRES, TYPE));
-
-  private String userName;
-  private String principal;
-  private String type;
-  private long expires;
-  private String token;
-
   private AuthenticationToken() {
-    userName = null;
-    principal = null;
-    type = null;
-    expires = -1;
-    token = "ANONYMOUS";
-    generateToken();
+    super();
   }
 
-  private static final String ILLEGAL_ARG_MSG = " is NULL, empty or contains a '" + ATTR_SEPARATOR + "'";
+  private AuthenticationToken(AuthToken token) {
+    super(token.getUserName(), token.getName(), token.getType());
+    setExpires(token.getExpires());
+  }
 
   /**
    * Creates an authentication token.
@@ -77,25 +55,7 @@ public class AuthenticationToken implements Principal {
    * (<code>System.currentTimeMillis() + validityPeriod</code>).
    */
   public AuthenticationToken(String userName, String principal, String type) {
-    checkForIllegalArgument(userName, "userName");
-    checkForIllegalArgument(principal, "principal");
-    checkForIllegalArgument(type, "type");
-    this.userName = userName;
-    this.principal = principal;
-    this.type = type;
-    this.expires = -1;
-  }
-  
-  /**
-   * Check if the provided value is invalid. Throw an error if it is invalid, NOP otherwise.
-   * 
-   * @param value the value to check.
-   * @param name the parameter name to use in an error message if the value is invalid.
-   */
-  private static void checkForIllegalArgument(String value, String name) {
-    if (value == null || value.length() == 0 || value.contains(ATTR_SEPARATOR)) {
-      throw new IllegalArgumentException(name + ILLEGAL_ARG_MSG);
-    }
+    super(userName, principal, type);
   }
 
   /**
@@ -105,79 +65,17 @@ public class AuthenticationToken implements Principal {
    */
   public void setExpires(long expires) {
     if (this != AuthenticationToken.ANONYMOUS) {
-      this.expires = expires;
-      generateToken();
+      super.setExpires(expires);
     }
   }
 
   /**
-   * Generates the token.
-   */
-  private void generateToken() {
-    StringBuffer sb = new StringBuffer();
-    sb.append(USER_NAME).append("=").append(getUserName()).append(ATTR_SEPARATOR);
-    sb.append(PRINCIPAL).append("=").append(getName()).append(ATTR_SEPARATOR);
-    sb.append(TYPE).append("=").append(getType()).append(ATTR_SEPARATOR);
-    sb.append(EXPIRES).append("=").append(getExpires());
-    token = sb.toString();
-  }
-
-  /**
-   * Returns the user name.
+   * Returns true if the token has expired.
    *
-   * @return the user name.
-   */
-  public String getUserName() {
-    return userName;
-  }
-
-  /**
-   * Returns the principal name (this method name comes from the JDK {@link Principal} interface).
-   *
-   * @return the principal name.
-   */
-  @Override
-  public String getName() {
-    return principal;
-  }
-
-  /**
-   * Returns the authentication mechanism of the token.
-   *
-   * @return the authentication mechanism of the token.
-   */
-  public String getType() {
-    return type;
-  }
-
-  /**
-   * Returns the expiration time of the token.
-   *
-   * @return the expiration time of the token, in milliseconds since Epoc.
-   */
-  public long getExpires() {
-    return expires;
-  }
-
-  /**
-   * Returns if the token has expired.
-   *
-   * @return if the token has expired.
+   * @return true if the token has expired.
    */
   public boolean isExpired() {
-    return getExpires() != -1 && System.currentTimeMillis() > getExpires();
-  }
-
-  /**
-   * Returns the string representation of the token.
-   * <p>
-   * This string representation is parseable by the {@link #parse} method.
-   *
-   * @return the string representation of the token.
-   */
-  @Override
-  public String toString() {
-    return token;
+    return super.isExpired();
   }
 
   /**
@@ -191,40 +89,6 @@ public class AuthenticationToken implements Principal {
    * an authentication token.
    */
   public static AuthenticationToken parse(String tokenStr) throws AuthenticationException {
-    Map<String, String> map = split(tokenStr);
-    if (!map.keySet().equals(ATTRIBUTES)) {
-      throw new AuthenticationException("Invalid token string, missing attributes");
+    return new AuthenticationToken(AuthToken.parse(tokenStr));
   }
-    long expires = Long.parseLong(map.get(EXPIRES));
-    AuthenticationToken token = new AuthenticationToken(map.get(USER_NAME), map.get(PRINCIPAL), map.get(TYPE));
-    token.setExpires(expires);
-    return token;
-  }
-
-  /**
-   * Splits the string representation of a token into attributes pairs.
-   *
-   * @param tokenStr string representation of a token.
-   *
-   * @return a map with the attribute pairs of the token.
-   *
-   * @throws AuthenticationException thrown if the string representation of the token could not be broken into
-   * attribute pairs.
-   */
-  private static Map<String, String> split(String tokenStr) throws AuthenticationException {
-    Map<String, String> map = new HashMap<String, String>();
-    StringTokenizer st = new StringTokenizer(tokenStr, ATTR_SEPARATOR);
-    while (st.hasMoreTokens()) {
-      String part = st.nextToken();
-      int separator = part.indexOf('=');
-      if (separator == -1) {
-        throw new AuthenticationException("Invalid authentication token");
-      }
-      String key = part.substring(0, separator);
-      String value = part.substring(separator + 1);
-      map.put(key, value);
-    }
-    return map;
-  }
-
 }

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java

@@ -0,0 +1,218 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.util;
+
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import java.security.Principal;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+/**
+ */
+public class AuthToken implements Principal {
+
+  /**
+   * Constant that identifies an anonymous request.
+   */
+
+  private static final String ATTR_SEPARATOR = "&";
+  private static final String USER_NAME = "u";
+  private static final String PRINCIPAL = "p";
+  private static final String EXPIRES = "e";
+  private static final String TYPE = "t";
+
+  private final static Set<String> ATTRIBUTES =
+    new HashSet<String>(Arrays.asList(USER_NAME, PRINCIPAL, EXPIRES, TYPE));
+
+  private String userName;
+  private String principal;
+  private String type;
+  private long expires;
+  private String tokenStr;
+
+  protected AuthToken() {
+    userName = null;
+    principal = null;
+    type = null;
+    expires = -1;
+    tokenStr = "ANONYMOUS";
+    generateToken();
+  }
+
+  private static final String ILLEGAL_ARG_MSG = " is NULL, empty or contains a '" + ATTR_SEPARATOR + "'";
+
+  /**
+   * Creates an authentication token.
+   *
+   * @param userName user name.
+   * @param principal principal (commonly matches the user name, with Kerberos is the full/long principal
+   * name while the userName is the short name).
+   * @param type the authentication mechanism name.
+   * (<code>System.currentTimeMillis() + validityPeriod</code>).
+   */
+  public AuthToken(String userName, String principal, String type) {
+    checkForIllegalArgument(userName, "userName");
+    checkForIllegalArgument(principal, "principal");
+    checkForIllegalArgument(type, "type");
+    this.userName = userName;
+    this.principal = principal;
+    this.type = type;
+    this.expires = -1;
+  }
+  
+  /**
+   * Check if the provided value is invalid. Throw an error if it is invalid, NOP otherwise.
+   * 
+   * @param value the value to check.
+   * @param name the parameter name to use in an error message if the value is invalid.
+   */
+  protected static void checkForIllegalArgument(String value, String name) {
+    if (value == null || value.length() == 0 || value.contains(ATTR_SEPARATOR)) {
+      throw new IllegalArgumentException(name + ILLEGAL_ARG_MSG);
+    }
+  }
+
+  /**
+   * Sets the expiration of the token.
+   *
+   * @param expires expiration time of the token in milliseconds since the epoch.
+   */
+  public void setExpires(long expires) {
+    this.expires = expires;
+      generateToken();
+  }
+
+  /**
+   * Returns true if the token has expired.
+   *
+   * @return true if the token has expired.
+   */
+  public boolean isExpired() {
+    return getExpires() != -1 && System.currentTimeMillis() > getExpires();
+  }
+
+  /**
+   * Generates the token.
+   */
+  private void generateToken() {
+    StringBuffer sb = new StringBuffer();
+    sb.append(USER_NAME).append("=").append(getUserName()).append(ATTR_SEPARATOR);
+    sb.append(PRINCIPAL).append("=").append(getName()).append(ATTR_SEPARATOR);
+    sb.append(TYPE).append("=").append(getType()).append(ATTR_SEPARATOR);
+    sb.append(EXPIRES).append("=").append(getExpires());
+    tokenStr = sb.toString();
+  }
+
+  /**
+   * Returns the user name.
+   *
+   * @return the user name.
+   */
+  public String getUserName() {
+    return userName;
+  }
+
+  /**
+   * Returns the principal name (this method name comes from the JDK {@link Principal} interface).
+   *
+   * @return the principal name.
+   */
+  @Override
+  public String getName() {
+    return principal;
+  }
+
+  /**
+   * Returns the authentication mechanism of the token.
+   *
+   * @return the authentication mechanism of the token.
+   */
+  public String getType() {
+    return type;
+  }
+
+  /**
+   * Returns the expiration time of the token.
+   *
+   * @return the expiration time of the token, in milliseconds since Epoc.
+   */
+  public long getExpires() {
+    return expires;
+  }
+
+  /**
+   * Returns the string representation of the token.
+   * <p>
+   * This string representation is parseable by the {@link #parse} method.
+   *
+   * @return the string representation of the token.
+   */
+  @Override
+  public String toString() {
+    return tokenStr;
+  }
+
+  public static AuthToken parse(String tokenStr) throws AuthenticationException {
+    if (tokenStr.length() >= 2) {
+      // strip the \" at the two ends of the tokenStr
+      if (tokenStr.charAt(0) == '\"' &&
+          tokenStr.charAt(tokenStr.length()-1) == '\"') {
+        tokenStr = tokenStr.substring(1, tokenStr.length()-1);
+      }
+    } 
+    Map<String, String> map = split(tokenStr);
+    // remove the signature part, since client doesn't care about it
+    map.remove("s");
+
+    if (!map.keySet().equals(ATTRIBUTES)) {
+      throw new AuthenticationException("Invalid token string, missing attributes");
+    }
+    long expires = Long.parseLong(map.get(EXPIRES));
+    AuthToken token = new AuthToken(map.get(USER_NAME), map.get(PRINCIPAL), map.get(TYPE));
+    token.setExpires(expires);
+    return token;
+  }
+
+  /**
+   * Splits the string representation of a token into attributes pairs.
+   *
+   * @param tokenStr string representation of a token.
+   *
+   * @return a map with the attribute pairs of the token.
+   *
+   * @throws AuthenticationException thrown if the string representation of the token could not be broken into
+   * attribute pairs.
+   */
+  private static Map<String, String> split(String tokenStr) throws AuthenticationException {
+    Map<String, String> map = new HashMap<String, String>();
+    StringTokenizer st = new StringTokenizer(tokenStr, ATTR_SEPARATOR);
+    while (st.hasMoreTokens()) {
+      String part = st.nextToken();
+      int separator = part.indexOf('=');
+      if (separator == -1) {
+        throw new AuthenticationException("Invalid authentication token");
+      }
+      String key = part.substring(0, separator);
+      String value = part.substring(separator + 1);
+      map.put(key, value);
+    }
+    return map;
+  }
+
+}

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationToken.java

@@ -13,7 +13,6 @@
  */
 package org.apache.hadoop.security.authentication.server;
 
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -28,103 +27,4 @@ public class TestAuthenticationToken {
     Assert.assertEquals(-1, AuthenticationToken.ANONYMOUS.getExpires());
     Assert.assertFalse(AuthenticationToken.ANONYMOUS.isExpired());
   }
-
-  @Test
-  public void testConstructor() throws Exception {
-    try {
-      new AuthenticationToken(null, "p", "t");
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    try {
-      new AuthenticationToken("", "p", "t");
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    try {
-      new AuthenticationToken("u", null, "t");
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    try {
-      new AuthenticationToken("u", "", "t");
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    try {
-      new AuthenticationToken("u", "p", null);
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    try {
-      new AuthenticationToken("u", "p", "");
-      Assert.fail();
-    } catch (IllegalArgumentException ex) {
-      // Expected
-    } catch (Throwable ex) {
-      Assert.fail();
-    }
-    new AuthenticationToken("u", "p", "t");
-  }
-
-  @Test
-  public void testGetters() throws Exception {
-    long expires = System.currentTimeMillis() + 50;
-    AuthenticationToken token = new AuthenticationToken("u", "p", "t");
-    token.setExpires(expires);
-    Assert.assertEquals("u", token.getUserName());
-    Assert.assertEquals("p", token.getName());
-    Assert.assertEquals("t", token.getType());
-    Assert.assertEquals(expires, token.getExpires());
-    Assert.assertFalse(token.isExpired());
-    Thread.sleep(51);
-    Assert.assertTrue(token.isExpired());
-  }
-
-  @Test
-  public void testToStringAndParse() throws Exception {
-    long expires = System.currentTimeMillis() + 50;
-    AuthenticationToken token = new AuthenticationToken("u", "p", "t");
-    token.setExpires(expires);
-    String str = token.toString();
-    token = AuthenticationToken.parse(str);
-    Assert.assertEquals("p", token.getName());
-    Assert.assertEquals("t", token.getType());
-    Assert.assertEquals(expires, token.getExpires());
-    Assert.assertFalse(token.isExpired());
-    Thread.sleep(51);
-    Assert.assertTrue(token.isExpired());
-  }
-
-  @Test
-  public void testParseInvalid() throws Exception {
-    long expires = System.currentTimeMillis() + 50;
-    AuthenticationToken token = new AuthenticationToken("u", "p", "t");
-    token.setExpires(expires);
-    String str = token.toString();
-    str = str.substring(0, str.indexOf("e="));
-    try {
-      AuthenticationToken.parse(str);
-      Assert.fail();
-    } catch (AuthenticationException ex) {
-      // Expected
-    } catch (Exception ex) {
-      Assert.fail();
-    }
-  }
 }

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestAuthToken.java

@@ -0,0 +1,127 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.util;
+
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAuthToken {
+
+  @Test
+  public void testConstructor() throws Exception {
+    try {
+      new AuthToken(null, "p", "t");
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    try {
+      new AuthToken("", "p", "t");
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    try {
+      new AuthToken("u", null, "t");
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    try {
+      new AuthToken("u", "", "t");
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    try {
+      new AuthToken("u", "p", null);
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    try {
+      new AuthToken("u", "p", "");
+      Assert.fail();
+    } catch (IllegalArgumentException ex) {
+      // Expected
+    } catch (Throwable ex) {
+      Assert.fail();
+    }
+    new AuthToken("u", "p", "t");
+  }
+
+  @Test
+  public void testGetters() throws Exception {
+    long expires = System.currentTimeMillis() + 50;
+    AuthToken token = new AuthToken("u", "p", "t");
+    token.setExpires(expires);
+    Assert.assertEquals("u", token.getUserName());
+    Assert.assertEquals("p", token.getName());
+    Assert.assertEquals("t", token.getType());
+    Assert.assertEquals(expires, token.getExpires());
+    Assert.assertFalse(token.isExpired());
+    Thread.sleep(70);               // +20 msec fuzz for timer granularity.
+    Assert.assertTrue(token.isExpired());
+  }
+
+  @Test
+  public void testToStringAndParse() throws Exception {
+    long expires = System.currentTimeMillis() + 50;
+    AuthToken token = new AuthToken("u", "p", "t");
+    token.setExpires(expires);
+    String str = token.toString();
+    token = AuthToken.parse(str);
+    Assert.assertEquals("p", token.getName());
+    Assert.assertEquals("t", token.getType());
+    Assert.assertEquals(expires, token.getExpires());
+    Assert.assertFalse(token.isExpired());
+    Thread.sleep(70);               // +20 msec fuzz for timer granularity.
+    Assert.assertTrue(token.isExpired());
+  }
+
+  @Test
+  public void testParseValidAndInvalid() throws Exception {
+    long expires = System.currentTimeMillis() + 50;
+    AuthToken token = new AuthToken("u", "p", "t");
+    token.setExpires(expires);
+    String ostr = token.toString();
+
+    String str1 = "\"" + ostr + "\"";
+    AuthToken.parse(str1);
+    
+    String str2 = ostr + "&s=1234";
+    AuthToken.parse(str2);
+
+    String str = ostr.substring(0, ostr.indexOf("e="));
+    try {
+      AuthToken.parse(str);
+      Assert.fail();
+    } catch (AuthenticationException ex) {
+      // Expected
+    } catch (Exception ex) {
+      Assert.fail();
+    }
+  }
+}

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -531,6 +531,9 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11497. Fix typo in ClusterSetup.html#Hadoop_Startup(Christian 
     Winkler via ozawa).
 
+    HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster.
+    (yzhangal via rkanter)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES