remove commons codec dependency

This commit removes the commons codec dependency and simplifies the
hasher code by only supporting bcrypt encryption in the users file. All tests
now also exercise the esusers realm with bcrypt instead of plain text passwords.

Closes elastic/elasticsearch#806

Original commit: elastic/x-pack-elasticsearch@3119267851

pom.xml

@@ -127,11 +127,6 @@
             <version>${license.plugin.version}</version>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>1.10</version>
-        </dependency>
         <dependency>
             <groupId>dk.brics.automaton</groupId>
             <artifactId>automaton</artifactId>

src/main/assemblies/plugin.xml

@@ -23,7 +23,6 @@
             <useTransitiveDependencies>false</useTransitiveDependencies>
             <includes>
                 <include>org.elasticsearch:elasticsearch-shield</include>
-                <include>commons-codec:commons-codec</include>
                 <include>dk.brics.automaton:automaton</include>
                 <include>com.unboundid:unboundid-ldapsdk</include>
             </includes>

src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.shield.authc;
 
-import org.apache.commons.codec.binary.Base64;
+import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -115,8 +115,8 @@ public class InternalAuthenticationService extends AbstractComponent implements
     }
 
     static User decodeUser(String text) {
-        byte[] bytes = Base64.decodeBase64(text);
         try {
+            byte[] bytes = Base64.decode(text);
             StreamInput input = StreamInput.wrap(bytes);
             return User.readFrom(input);
         } catch (IOException ioe) {
@@ -129,7 +129,7 @@ public class InternalAuthenticationService extends AbstractComponent implements
             BytesStreamOutput output = new BytesStreamOutput();
             User.writeTo(user, output);
             byte[] bytes = output.bytes().toBytes();
-            return Base64.encodeBase64String(bytes);
+            return Base64.encodeBytes(bytes);
         } catch (IOException ioe) {
             if (logger != null) {
                 logger.error("could not encode authenticated user in message header... falling back to token headers", ioe);

src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java

@@ -43,7 +43,7 @@ public class FileUserPasswdStore {
     private final ESLogger logger;
 
     private final Path file;
-    final Hasher hasher = Hasher.HTPASSWD;
+    final Hasher hasher = Hasher.BCRYPT;
 
     private volatile ImmutableMap<String, char[]> users;
 

src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java

@@ -147,7 +147,7 @@ public class ESUsersTool extends CliTool {
                 terminal.println("User [%s] already exists", username);
                 return ExitStatus.CODE_ERROR;
             }
-            Hasher hasher = Hasher.HTPASSWD;
+            Hasher hasher = Hasher.BCRYPT;
             users.put(username, hasher.hash(passwd));
             FileUserPasswdStore.writeFile(users, file);
 
@@ -289,7 +289,7 @@ public class ESUsersTool extends CliTool {
                 terminal.println("User [%s] doesn't exist", username);
                 return ExitStatus.NO_USER;
             }
-            Hasher hasher = Hasher.HTPASSWD;
+            Hasher hasher = Hasher.BCRYPT;
             users.put(username, hasher.hash(passwd));
             FileUserPasswdStore.writeFile(users, file);
             return ExitStatus.OK;

src/main/java/org/elasticsearch/shield/authc/support/Hasher.java

@@ -5,9 +5,7 @@
  */
 package org.elasticsearch.shield.authc.support;
 
-import org.apache.commons.codec.binary.Base64;
+import org.elasticsearch.common.Base64;
-import org.apache.commons.codec.digest.*;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.base.Charsets;
 import org.elasticsearch.shield.ShieldException;
 import org.elasticsearch.shield.ShieldSettingsException;
@@ -23,47 +21,6 @@ import java.util.concurrent.ThreadLocalRandom;
  */
 public enum Hasher {
 
-    /**
-     * A hasher that is compatible with apache htpasswd. Hashes by default using bcrypt type $2a$
-     * but can verify any of the hashes supported by htpasswd.
-     */
-    HTPASSWD() {
-
-        @Override
-        public char[] hash(SecuredString text) {
-            String salt = org.elasticsearch.shield.authc.support.BCrypt.gensalt();
-            return BCrypt.hashpw(text, salt).toCharArray();
-        }
-
-        @Override
-        public boolean verify(SecuredString text, char[] hash) {
-            String hashStr = new String(hash);
-            if (hashStr.startsWith(BCRYPT_PREFIX)) {
-                return BCrypt.checkpw(text, hashStr);
-            }
-            if (hashStr.startsWith(PLAIN_PREFIX)) {
-                hashStr = hashStr.substring(PLAIN_PREFIX.length());
-                return SecuredString.constantTimeEquals(text, hashStr);
-            }
-            byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            if (hashStr.startsWith(APR1_PREFIX)) {
-                return SecuredString.constantTimeEquals(hashStr, Md5Crypt.apr1Crypt(textBytes, hashStr));
-            }
-            if (hashStr.startsWith(SHA1_PREFIX)) {
-                String passwd64 = Base64.encodeBase64String(DigestUtils.sha1(textBytes));
-                String hashNoPrefix = hashStr.substring(SHA1_PREFIX.length());
-                return SecuredString.constantTimeEquals(passwd64, hashNoPrefix);
-            }
-            if (hashStr.startsWith(SHA2_PREFIX_5) || hashStr.startsWith(SHA2_PREFIX_6)) {
-                return SecuredString.constantTimeEquals(hashStr, Sha2Crypt.sha256Crypt(textBytes, hashStr));
-            }
-            return CRYPT_SUPPORTED ?
-                    SecuredString.constantTimeEquals(hashStr, Crypt.crypt(textBytes, hashStr)) :  // crypt algo
-                    SecuredString.constantTimeEquals(text, hashStr);                              // plain text
-        }
-    },
-
-
     BCRYPT() {
         @Override
         public char[] hash(SecuredString text) {
@@ -183,31 +140,13 @@ public enum Hasher {
         }
     },
 
-    APR1() {
-        @Override
-        public char[] hash(SecuredString text) {
-            byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            return Md5Crypt.apr1Crypt(textBytes).toCharArray();
-        }
-
-        @Override
-        public boolean verify(SecuredString text, char[] hash) {
-            String hashStr = new String(hash);
-            if (!hashStr.startsWith(APR1_PREFIX)) {
-                return false;
-            }
-            byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            return SecuredString.constantTimeEquals(hashStr, Md5Crypt.apr1Crypt(textBytes, hashStr));
-        }
-    },
-
     SHA1() {
         @Override
         public char[] hash(SecuredString text) {
             byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
             MessageDigest md = SHA1Provider.sha1();
             md.update(textBytes);
-            String hash = Base64.encodeBase64String(md.digest());
+            String hash = Base64.encodeBytes(md.digest());
             return (SHA1_PREFIX + hash).toCharArray();
         }
 
@@ -220,36 +159,18 @@ public enum Hasher {
             byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
             MessageDigest md = SHA1Provider.sha1();
             md.update(textBytes);
-            String passwd64 = Base64.encodeBase64String(md.digest());
+            String passwd64 = Base64.encodeBytes(md.digest());
             String hashNoPrefix = hashStr.substring(SHA1_PREFIX.length());
             return SecuredString.constantTimeEquals(hashNoPrefix, passwd64);
         }
     },
 
-    SHA2() {
-        @Override
-        public char[] hash(SecuredString text) {
-            byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            return Sha2Crypt.sha256Crypt(textBytes).toCharArray();
-        }
-
-        @Override
-        public boolean verify(SecuredString text, char[] hash) {
-            String hashStr = new String(hash);
-            if (hashStr.startsWith(SHA2_PREFIX_5) || hashStr.startsWith(SHA2_PREFIX_6)) {
-                byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-                return SecuredString.constantTimeEquals(hashStr, Sha2Crypt.sha256Crypt(textBytes, hashStr));
-            }
-            return false;
-        }
-    },
-
     MD5() {
         @Override
         public char[] hash(SecuredString text) {
             MessageDigest md = MD5Provider.md5();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
-            String hash = Base64.encodeBase64String(md.digest());
+            String hash = Base64.encodeBytes(md.digest());
             return (MD5_PREFIX + hash).toCharArray();
         }
 
@@ -262,7 +183,7 @@ public enum Hasher {
             hashStr = hashStr.substring(MD5_PREFIX.length());
             MessageDigest md = MD5Provider.md5();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
-            String computedHashStr = Base64.encodeBase64String(md.digest());
+            String computedHashStr = Base64.encodeBytes(md.digest());
             return SecuredString.constantTimeEquals(hashStr, computedHashStr);
         }
     },
@@ -274,7 +195,7 @@ public enum Hasher {
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             char[] salt = SaltProvider.salt(8);
             md.update(CharArrays.toUtf8Bytes(salt));
-            String hash = Base64.encodeBase64String(md.digest());
+            String hash = Base64.encodeBytes(md.digest());
             char[] result = new char[SSHA256_PREFIX.length() + salt.length + hash.length()];
             System.arraycopy(SSHA256_PREFIX.toCharArray(), 0, result, 0, SSHA256_PREFIX.length());
             System.arraycopy(salt, 0, result, SSHA256_PREFIX.length(), salt.length);
@@ -293,7 +214,7 @@ public enum Hasher {
             MessageDigest md = SHA256Provider.sha256();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             md.update(new String(saltAndHash, 0, 8).getBytes(Charsets.UTF_8));
-            String computedHash = Base64.encodeBase64String(md.digest());
+            String computedHash = Base64.encodeBytes(md.digest());
             return SecuredString.constantTimeEquals(computedHash, new String(saltAndHash, 8, saltAndHash.length - 8));
         }
     },
@@ -310,22 +231,16 @@ public enum Hasher {
         }
     };
 
-    private static final String APR1_PREFIX = "$apr1$";
     private static final String BCRYPT_PREFIX = "$2a$";
     private static final String SHA1_PREFIX = "{SHA}";
-    private static final String SHA2_PREFIX_5 = "$5$";
-    private static final String SHA2_PREFIX_6 = "$6$";
     private static final String MD5_PREFIX = "{MD5}";
     private static final String SSHA256_PREFIX = "{SSHA256}";
-    private static final String PLAIN_PREFIX = "{plain}";
-    static final boolean CRYPT_SUPPORTED = !Constants.WINDOWS;
 
     public static Hasher resolve(String name, Hasher defaultHasher) {
         if (name == null) {
             return defaultHasher;
         }
         switch (name.toLowerCase(Locale.ROOT)) {
-            case "htpasswd"     : return HTPASSWD;
             case "bcrypt"       : return BCRYPT;
             case "bcrypt4"      : return BCRYPT4;
             case "bcrypt5"      : return BCRYPT5;
@@ -333,9 +248,7 @@ public enum Hasher {
             case "bcrypt7"      : return BCRYPT7;
             case "bcrypt8"      : return BCRYPT8;
             case "bcrypt9"      : return BCRYPT9;
-            case "apr1"         : return APR1;
             case "sha1"         : return SHA1;
-            case "sha2"         : return SHA2;
             case "md5"          : return MD5;
             case "ssha256"      : return SSHA256;
             case "noop"         :
@@ -363,9 +276,9 @@ public enum Hasher {
 
         static {
             try {
-                digest = MessageDigest.getInstance(MessageDigestAlgorithms.MD5);
+                digest = MessageDigest.getInstance("MD5");
             } catch (NoSuchAlgorithmException e) {
-                throw new ShieldException("unsupported digest algorithm [" + MessageDigestAlgorithms.MD5 + "]. Please verify you are running on Java 7 or above", e);
+                throw new ShieldException("unsupported digest algorithm [MD5]. Please verify you are running on Java 7 or above", e);
             }
         }
 
@@ -386,9 +299,9 @@ public enum Hasher {
 
         static {
             try {
-                digest = MessageDigest.getInstance(MessageDigestAlgorithms.SHA_1);
+                digest = MessageDigest.getInstance("SHA-1");
             } catch (NoSuchAlgorithmException e) {
-                throw new ShieldException("unsupported digest algorithm [" + MessageDigestAlgorithms.SHA_1 + "]", e);
+                throw new ShieldException("unsupported digest algorithm [SHA-1]", e);
             }
         }
 
@@ -398,7 +311,7 @@ public enum Hasher {
                 sha1.reset();
                 return sha1;
             } catch (CloneNotSupportedException e) {
-                throw new ShieldException("could not create SHA1 digest", e);
+                throw new ShieldException("could not create SHA-1 digest", e);
             }
         }
     }
@@ -409,9 +322,9 @@ public enum Hasher {
 
         static {
             try {
-                digest = MessageDigest.getInstance(MessageDigestAlgorithms.SHA_256);
+                digest = MessageDigest.getInstance("SHA-256");
             } catch (NoSuchAlgorithmException e) {
-                throw new ShieldException("unsupported digest algorithm [" + MessageDigestAlgorithms.SHA_256 + "]. Please verify you are running on Java 7 or above", e);
+                throw new ShieldException("unsupported digest algorithm [SHA-256]. Please verify you are running on Java 7 or above", e);
             }
         }
 
@@ -421,7 +334,7 @@ public enum Hasher {
                 sha.reset();
                 return sha;
             } catch (CloneNotSupportedException e) {
-                throw new ShieldException("could not create [" + MessageDigestAlgorithms.SHA_256 + "] digest", e);
+                throw new ShieldException("could not create [SHA-256] digest", e);
             }
         }
     }

src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordToken.java

@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.shield.authc.support;
 
-import org.apache.commons.codec.binary.Base64;
+import org.elasticsearch.common.Base64;
-import org.elasticsearch.common.base.Charsets;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.shield.authc.AuthenticationException;
 import org.elasticsearch.shield.authc.AuthenticationToken;
 import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.transport.TransportRequest;
 
+import java.io.IOException;
 import java.nio.CharBuffer;
 import java.util.Arrays;
 import java.util.Objects;
@@ -72,20 +72,7 @@ public class UsernamePasswordToken implements AuthenticationToken {
             return defaultToken;
         }
 
-        Matcher matcher = BASIC_AUTH_PATTERN.matcher(authStr.trim());
+        return extractToken(authStr);
-        if (!matcher.matches()) {
-            throw new AuthenticationException("invalid basic authentication header value");
-        }
-
-        char[] userpasswd = CharArrays.utf8BytesToChars(Base64.decodeBase64(matcher.group(1)));
-        int i = CharArrays.indexOf(userpasswd, ':');
-        if (i < 0) {
-            throw new AuthenticationException("invalid basic authentication header value");
-        }
-
-        return new UsernamePasswordToken(
-                new String(Arrays.copyOfRange(userpasswd, 0, i)),
-                new SecuredString(Arrays.copyOfRange(userpasswd, i + 1, userpasswd.length)));
     }
 
     public static UsernamePasswordToken extractToken(RestRequest request, UsernamePasswordToken defaultToken) {
@@ -94,12 +81,22 @@ public class UsernamePasswordToken implements AuthenticationToken {
             return defaultToken;
         }
 
-        Matcher matcher = BASIC_AUTH_PATTERN.matcher(authStr.trim());
+        return extractToken(authStr);
+    }
+
+    static UsernamePasswordToken extractToken(String token) {
+        Matcher matcher = BASIC_AUTH_PATTERN.matcher(token.trim());
         if (!matcher.matches()) {
             throw new AuthenticationException("invalid basic authentication header value");
         }
 
-        char[] userpasswd = CharArrays.utf8BytesToChars(Base64.decodeBase64(matcher.group(1)));
+        char[] userpasswd;
+        try {
+            userpasswd = CharArrays.utf8BytesToChars(Base64.decode(matcher.group(1)));
+        } catch (IllegalArgumentException|IOException e) {
+            throw new AuthenticationException("invalid basic authentication header encoding", e);
+        }
+
         int i = CharArrays.indexOf(userpasswd, ':');
         if (i < 0) {
             throw new AuthenticationException("invalid basic authentication header value");
@@ -119,7 +116,7 @@ public class UsernamePasswordToken implements AuthenticationToken {
         chars.put(username).put(':').put(passwd.internalChars());
 
         //TODO we still have passwords in Strings in headers
-        String basicToken = new String(Base64.encodeBase64(CharArrays.toUtf8Bytes(chars.array())), Charsets.UTF_8);
+        String basicToken = Base64.encodeBytes(CharArrays.toUtf8Bytes(chars.array()));
         return "Basic " + basicToken;
     }
 }

src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java

@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.shield.crypto;
 
-import org.apache.commons.codec.binary.Base64;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.base.Charsets;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -191,7 +191,7 @@ public class InternalCryptoService extends AbstractLifecycleComponent<InternalCr
         }
 
         byte[] charBytes = CharArrays.toUtf8Bytes(chars);
-        return Base64.encodeBase64String(encryptInternal(charBytes, key)).toCharArray();
+        return Base64.encodeBytes(encryptInternal(charBytes, key)).toCharArray();
     }
 
     @Override
@@ -212,7 +212,12 @@ public class InternalCryptoService extends AbstractLifecycleComponent<InternalCr
                     + "the file [" + ShieldPlugin.resolveConfigFile(env, FILE_NAME) + "] to all nodes and the key will be loaded automatically.");
         }
 
-        byte[] bytes = Base64.decodeBase64(new String(chars));
+        byte[] bytes;
+        try {
+            bytes = Base64.decode(new String(chars));
+        } catch (IOException e) {
+            throw new ShieldException("unable to decode encrypted data", e);
+        }
         byte[] decrypted = decryptInternal(bytes, key);
         return CharArrays.utf8BytesToChars(decrypted);
     }
@@ -274,7 +279,11 @@ public class InternalCryptoService extends AbstractLifecycleComponent<InternalCr
     private String signInternal(String text) {
         Mac mac = createMac(systemKey);
         byte[] sig = mac.doFinal(text.getBytes(Charsets.UTF_8));
-        return Base64.encodeBase64URLSafeString(sig);
+        try {
+            return Base64.encodeBytes(sig, 0, sig.length, Base64.URL_SAFE);
+        } catch (IOException e) {
+            throw new SignatureException("unable to encode signed data", e);
+        }
     }
 
 

src/test/java/org/elasticsearch/bench/HasherBenchmark.java

@@ -28,7 +28,6 @@ public class HasherBenchmark {
         test(Hasher.SSHA256).print();
         test(Hasher.MD5).print();
         test(Hasher.SHA1).print();
-        test(Hasher.APR1).print();
         test(Hasher.BCRYPT4).print();
     }
 

src/test/java/org/elasticsearch/integration/AbstractPrivilegeTests.java

@@ -10,6 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ShieldIntegrationTest;
@@ -28,6 +29,8 @@ import static org.hamcrest.Matchers.*;
  */
 public abstract class AbstractPrivilegeTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
+
     private CloseableHttpClient httpClient = HttpClients.createDefault();
 
     @After

src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java

@@ -12,6 +12,8 @@ import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequest;
 import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheResponse;
 import org.elasticsearch.shield.authc.Realm;
 import org.elasticsearch.shield.authc.Realms;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.client.ShieldClient;
@@ -32,6 +34,8 @@ import static org.hamcrest.Matchers.*;
  */
 public class ClearRealmsCacheTests extends ShieldIntegrationTest {
 
+    private static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
+
     private static String[] usernames;
 
     @BeforeClass
@@ -96,7 +100,7 @@ public class ClearRealmsCacheTests extends ShieldIntegrationTest {
     protected String configUsers() {
         StringBuilder builder = new StringBuilder(ShieldSettingsSource.CONFIG_STANDARD_USER);
         for (String username : usernames) {
-            builder.append(username).append(":{plain}passwd\n");
+            builder.append(username).append(":").append(USERS_PASSWD_HASHED).append("\n");
         }
         return builder.toString();
     }

src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java

@@ -37,9 +37,9 @@ public class ClusterPrivilegeTests extends AbstractPrivilegeTests {
                     "    'someindex': all\n";
 
     public static final String USERS =
-                    "user_a:{plain}passwd\n" +
+                    "user_a:" + USERS_PASSWD_HASHED + "\n" +
-                    "user_b:{plain}passwd\n" +
+                    "user_b:" + USERS_PASSWD_HASHED + "\n" +
-                    "user_c:{plain}passwd\n";
+                    "user_c:" + USERS_PASSWD_HASHED + "\n";
 
     public static final String USERS_ROLES =
                     "role_a:user_a\n" +

src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java

@@ -76,22 +76,22 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTests {
                     "\n";
 
     public static final String USERS =
-            "admin:{plain}passwd\n" +
+            "admin:" + USERS_PASSWD_HASHED + "\n" +
-            "u1:{plain}passwd\n" +
+            "u1:" + USERS_PASSWD_HASHED + "\n" +
-            "u2:{plain}passwd\n" +
+            "u2:" + USERS_PASSWD_HASHED + "\n" +
-            "u3:{plain}passwd\n" +
+            "u3:" + USERS_PASSWD_HASHED + "\n" +
-            "u4:{plain}passwd\n" +
+            "u4:" + USERS_PASSWD_HASHED + "\n" +
-            "u5:{plain}passwd\n" +
+            "u5:" + USERS_PASSWD_HASHED + "\n" +
-            "u6:{plain}passwd\n" +
+            "u6:" + USERS_PASSWD_HASHED + "\n" +
-            "u7:{plain}passwd\n" +
+            "u7:" + USERS_PASSWD_HASHED + "\n"+
-            "u8:{plain}passwd\n" +
+            "u8:" + USERS_PASSWD_HASHED + "\n"+
-            "u9:{plain}passwd\n" +
+            "u9:" + USERS_PASSWD_HASHED + "\n" +
-            "u10:{plain}passwd\n" +
+            "u10:" + USERS_PASSWD_HASHED + "\n" +
-            "u11:{plain}passwd\n" +
+            "u11:" + USERS_PASSWD_HASHED + "\n" +
-            "u12:{plain}passwd\n" +
+            "u12:" + USERS_PASSWD_HASHED + "\n" +
-            "u13:{plain}passwd\n" +
+            "u13:" + USERS_PASSWD_HASHED + "\n" +
-            "u14:{plain}passwd\n" +
+            "u14:" + USERS_PASSWD_HASHED + "\n" +
-            "u15:{plain}passwd\n";
+            "u15:" + USERS_PASSWD_HASHED + "\n";
 
     public static final String USERS_ROLES =
             "all_indices_role:admin,u8\n" +

src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java

@@ -9,6 +9,8 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.authz.AuthorizationException;
@@ -26,6 +28,8 @@ import static org.hamcrest.Matchers.is;
 
 public class MultipleIndicesPermissionsTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
+
     @Override
     protected String configRoles() {
         return ShieldSettingsSource.DEFAULT_ROLE + ":\n" +
@@ -48,8 +52,8 @@ public class MultipleIndicesPermissionsTests extends ShieldIntegrationTest {
     @Override
     protected String configUsers() {
         return ShieldSettingsSource.CONFIG_STANDARD_USER +
-                "user_a:{plain}passwd\n" +
+                "user_a:" + USERS_PASSWD_HASHED + "\n" +
-                "user_ab:{plain}passwd\n";
+                "user_ab:" + USERS_PASSWD_HASHED + "\n";
     }
 
     @Override

src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java

@@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
@@ -31,6 +32,8 @@ import static org.hamcrest.Matchers.hasSize;
  */
 public class PermissionPrecedenceTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray())));
+
     @Override
     protected String configRoles() {
         return "admin:\n" +
@@ -50,9 +53,9 @@ public class PermissionPrecedenceTests extends ShieldIntegrationTest {
 
     @Override
     protected String configUsers() {
-        return "admin:{plain}test123\n" +
+        return "admin:" + USERS_PASSWD_HASHED + "\n" +
-                "client:{plain}test123\n" +
+                "client:" + USERS_PASSWD_HASHED + "\n" +
-                "user:{plain}test123\n";
+                "user:" + USERS_PASSWD_HASHED + "\n";
     }
 
     @Override

src/test/java/org/elasticsearch/integration/SearchGetAndSuggestPermissionsTests.java

@@ -12,6 +12,8 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.suggest.SuggestResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.search.suggest.SuggestBuilders;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.authz.AuthorizationException;
@@ -26,6 +28,8 @@ import static org.hamcrest.Matchers.is;
 
 public class SearchGetAndSuggestPermissionsTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
+
     @Override
     protected String configRoles() {
         return super.configRoles() + "\n" +
@@ -46,9 +50,9 @@ public class SearchGetAndSuggestPermissionsTests extends ShieldIntegrationTest {
     @Override
     protected String configUsers() {
         return super.configUsers() +
-                "search_user:{plain}passwd\n" +
+                "search_user:" + USERS_PASSWD_HASHED + "\n" +
-                "get_user:{plain}passwd\n" +
+                "get_user:" + USERS_PASSWD_HASHED + "\n" +
-                "suggest_user:{plain}passwd\n";
+                "suggest_user:" + USERS_PASSWD_HASHED + "\n";
 
     }
 

src/test/java/org/elasticsearch/integration/ShieldClearScrollTests.java

@@ -11,6 +11,7 @@ import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authz.AuthorizationException;
 import org.elasticsearch.test.ShieldIntegrationTest;
@@ -28,13 +29,15 @@ import static org.hamcrest.Matchers.is;
 
 public class ShieldClearScrollTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("change_me".toCharArray())));
+
     private List<String> scrollIds;
 
     @Override
     protected String configUsers() {
         return super.configUsers() +
-            "allowed_user:{plain}change_me\n" +
+            "allowed_user:" + USERS_PASSWD_HASHED + "\n" +
-            "denied_user:{plain}change_me\n" ;
+            "denied_user:" + USERS_PASSWD_HASHED + "\n" ;
     }
 
     @Override

src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java

@@ -109,7 +109,7 @@ public class FileUserPasswdStoreTests extends ElasticsearchTestCase {
 
         try (BufferedWriter writer = Files.newBufferedWriter(tmp, Charsets.UTF_8, StandardOpenOption.APPEND)) {
             writer.newLine();
-            writer.append("foobar:").append(new String(Hasher.HTPASSWD.hash(SecuredStringTests.build("barfoo"))));
+            writer.append("foobar:").append(new String(Hasher.BCRYPT.hash(SecuredStringTests.build("barfoo"))));
         }
 
         if (!latch.await(5, TimeUnit.SECONDS)) {

src/test/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersToolTests.java

@@ -121,7 +121,7 @@ public class ESUsersToolTests extends CliToolTestCase {
         String line = lines.get(0);
         assertThat(line, startsWith("user1:"));
         String hash = line.substring("user1:".length());
-        assertThat(Hasher.HTPASSWD.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
+        assertThat(Hasher.BCRYPT.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
 
         assertFileExists(userRolesFile);
         lines = Files.readAllLines(userRolesFile, Charsets.UTF_8);
@@ -156,7 +156,7 @@ public class ESUsersToolTests extends CliToolTestCase {
         for (String line : lines) {
             if (line.startsWith("user1")) {
                 String hash = line.substring("user1:".length());
-                assertThat(Hasher.HTPASSWD.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
+                assertThat(Hasher.BCRYPT.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
             }
         }
 
@@ -357,7 +357,7 @@ public class ESUsersToolTests extends CliToolTestCase {
         String line = lines.get(0);
         assertThat(line, startsWith("user1:"));
         String hash = line.substring("user1:".length());
-        assertThat(Hasher.HTPASSWD.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
+        assertThat(Hasher.BCRYPT.verify(SecuredStringTests.build("changeme"), hash.toCharArray()), is(true));
     }
 
     @Test

src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java

@@ -30,7 +30,7 @@ public class CachingUsernamePasswordRealmTests extends ElasticsearchTestCase {
     @Test
     public void testSettings() throws Exception {
 
-        String hashAlgo = randomFrom("bcrypt", "bcrypt4", "bcrypt5", "bcrypt6", "bcrypt7", "bcrypt8", "bcrypt9", "sha1", "sha2", "md5", "clear_text", "noop");
+        String hashAlgo = randomFrom("bcrypt", "bcrypt4", "bcrypt5", "bcrypt6", "bcrypt7", "bcrypt8", "bcrypt9", "sha1", "ssha256", "md5", "clear_text", "noop");
         int maxUsers = randomIntBetween(10, 100);
         TimeValue ttl = TimeValue.timeValueMinutes(randomIntBetween(10, 20));
         Settings settings = Settings.builder()

src/test/java/org/elasticsearch/shield/authc/support/HasherTests.java

@@ -16,28 +16,6 @@ import static org.hamcrest.Matchers.sameInstance;
  */
 public class HasherTests extends ElasticsearchTestCase {
 
-    @Test
-    public void testHtpasswd_ToolGenerated() throws Exception {
-        Hasher hasher = Hasher.HTPASSWD;
-        SecuredString passwd = SecuredStringTests.build("test123");
-        assertTrue(hasher.verify(passwd, "$2a$05$zxnP0vdREMxnEpkLCDI2OuSaSk/QEKA2.A42iOpI6U2u.RLLOWm1e".toCharArray()));
-        assertTrue(hasher.verify(passwd, "$2a$10$vNMk6GyVUU./7YSZB6BGPuozm921GVPw/Pdukzd09s.sL2rIWROU6".toCharArray()));
-        assertTrue(hasher.verify(passwd, "$apr1$R3DdqiAZ$aljIkaIVPSarmDMlJUBBP.".toCharArray()));
-        if (!Hasher.CRYPT_SUPPORTED) {
-            assertTrue(hasher.verify(passwd, "test123".toCharArray()));
-        } else {
-            assertTrue(hasher.verify(passwd, "hsP1PYSLsEEvs".toCharArray()));
-        }
-        assertTrue(hasher.verify(passwd, "{plain}test123".toCharArray()));
-        assertTrue(hasher.verify(passwd, "{SHA}cojt0Pw//L6ToM8G41aOKFIWh7w=".toCharArray()));
-        assertTrue(hasher.verify(passwd, "$5$RsqcsPiF$51tIIXf6oZb3Awox6FWNhITVlM/aW3oa8uN2eptIf54".toCharArray()));
-    }
-
-    @Test
-    public void testHtpasswd_SelfGenerated() throws Exception {
-        testHasherSelfGenerated(Hasher.HTPASSWD);
-    }
-
     @Test
     public void testBcryptFamily_SelfGenerated() throws Exception {
         testHasherSelfGenerated(Hasher.BCRYPT);
@@ -59,11 +37,6 @@ public class HasherTests extends ElasticsearchTestCase {
         testHasherSelfGenerated(Hasher.SHA1);
     }
 
-    @Test
-    public void testSha2_SelfGenerated() throws Exception {
-        testHasherSelfGenerated(Hasher.SHA2);
-    }
-
     @Test
     public void testSSHA256_SelfGenerated() throws Exception {
         testHasherSelfGenerated(Hasher.SSHA256);
@@ -76,7 +49,6 @@ public class HasherTests extends ElasticsearchTestCase {
 
     @Test
     public void testResolve() throws Exception {
-        assertThat(Hasher.resolve("htpasswd"), sameInstance(Hasher.HTPASSWD));
         assertThat(Hasher.resolve("bcrypt"), sameInstance(Hasher.BCRYPT));
         assertThat(Hasher.resolve("bcrypt4"), sameInstance(Hasher.BCRYPT4));
         assertThat(Hasher.resolve("bcrypt5"), sameInstance(Hasher.BCRYPT5));
@@ -85,8 +57,6 @@ public class HasherTests extends ElasticsearchTestCase {
         assertThat(Hasher.resolve("bcrypt8"), sameInstance(Hasher.BCRYPT8));
         assertThat(Hasher.resolve("bcrypt9"), sameInstance(Hasher.BCRYPT9));
         assertThat(Hasher.resolve("sha1"), sameInstance(Hasher.SHA1));
-        assertThat(Hasher.resolve("sha2"), sameInstance(Hasher.SHA2));
-        assertThat(Hasher.resolve("apr1"), sameInstance(Hasher.APR1));
         assertThat(Hasher.resolve("md5"), sameInstance(Hasher.MD5));
         assertThat(Hasher.resolve("ssha256"), sameInstance(Hasher.SSHA256));
         assertThat(Hasher.resolve("noop"), sameInstance(Hasher.NOOP));

src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java

@@ -6,7 +6,7 @@
 package org.elasticsearch.shield.authc.support;
 
 import com.google.common.base.Charsets;
-import org.apache.commons.codec.binary.Base64;
+import org.elasticsearch.common.Base64;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.shield.authc.AuthenticationException;
 import org.elasticsearch.test.ElasticsearchTestCase;
@@ -36,7 +36,7 @@ public class UsernamePasswordTokenTests extends ElasticsearchTestCase {
         assertThat(header, notNullValue());
         assertTrue(header.startsWith("Basic "));
         String token = header.substring("Basic ".length());
-        token = new String(Base64.decodeBase64(token), Charsets.UTF_8);
+        token = new String(Base64.decode(token), Charsets.UTF_8);
         int i = token.indexOf(":");
         assertTrue(i > 0);
         String username = token.substring(0, i);
@@ -48,7 +48,7 @@ public class UsernamePasswordTokenTests extends ElasticsearchTestCase {
     @Test
     public void testExtractToken() throws Exception {
         TransportRequest request = new TransportRequest() {};
-        String header = "Basic " + new String(Base64.encodeBase64("user1:test123".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
+        String header = "Basic " + Base64.encodeBytes("user1:test123".getBytes(Charsets.UTF_8));
         request.putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, header);
         UsernamePasswordToken token = UsernamePasswordToken.extractToken(request, null);
         assertThat(token, notNullValue());

src/test/java/org/elasticsearch/shield/authz/AnalyzeTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.shield.authz;
 
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegrationTest;
 import org.junit.Test;
@@ -15,11 +16,13 @@ import static org.hamcrest.CoreMatchers.containsString;
 
 public class AnalyzeTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray())));
+
     @Override
     protected String configUsers() {
         return super.configUsers() +
-                "analyze_indices:{plain}test123\n" +
+                "analyze_indices:" + USERS_PASSWD_HASHED + "\n" +
-                "analyze_cluster:{plain}test123\n";
+                "analyze_cluster:" + USERS_PASSWD_HASHED + "\n";
     }
 
     @Override

src/test/java/org/elasticsearch/shield/authz/IndexAliasesTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegrationTest;
 import org.junit.Before;
@@ -24,14 +25,16 @@ import static org.hamcrest.CoreMatchers.is;
 
 public class IndexAliasesTests extends ShieldIntegrationTest {
 
+    protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray())));
+
     @Override
     protected String configUsers() {
         return super.configUsers() +
-                "create_only:{plain}test123\n" +
+                "create_only:" + USERS_PASSWD_HASHED + "\n" +
-                "create_test_aliases_test:{plain}test123\n" +
+                "create_test_aliases_test:" + USERS_PASSWD_HASHED + "\n" +
-                "create_test_aliases_alias:{plain}test123\n" +
+                "create_test_aliases_alias:" + USERS_PASSWD_HASHED + "\n" +
-                "create_test_aliases_test_alias:{plain}test123\n" +
+                "create_test_aliases_test_alias:" + USERS_PASSWD_HASHED + "\n" +
-                "aliases_only:{plain}test123\n";
+                "aliases_only:" + USERS_PASSWD_HASHED + "\n";
     }
 
     @Override

src/test/java/org/elasticsearch/test/ShieldSettingsSource.java

@@ -13,6 +13,7 @@ import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
+import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.crypto.InternalCryptoService;
@@ -45,14 +46,15 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
 
     public static final String DEFAULT_USER_NAME = "test_user";
     public static final String DEFAULT_PASSWORD = "changeme";
+    private static final String DEFAULT_PASSWORD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString(DEFAULT_PASSWORD.toCharArray())));
     public static final String DEFAULT_ROLE = "user";
 
     public static final String DEFAULT_TRANSPORT_CLIENT_ROLE = "trans_client_user";
     public static final String DEFAULT_TRANSPORT_CLIENT_USER_NAME = "test_trans_client_user";
 
     public static final String CONFIG_STANDARD_USER =
-            DEFAULT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n" +
+            DEFAULT_USER_NAME + ":" + DEFAULT_PASSWORD_HASHED + "\n" +
-            DEFAULT_TRANSPORT_CLIENT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n";
+            DEFAULT_TRANSPORT_CLIENT_USER_NAME + ":" + DEFAULT_PASSWORD_HASHED + "\n";
 
     public static final String CONFIG_STANDARD_USER_ROLES =
             DEFAULT_ROLE + ":" + DEFAULT_USER_NAME + "," + DEFAULT_TRANSPORT_CLIENT_USER_NAME + "\n" +

test-signatures.txt

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 # Licensed to Elasticsearch under one or more contributor
 # license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright
@@ -18,3 +19,15 @@ com.carrotsearch.randomizedtesting.RandomizedTest#globalTempDir() @ Use newTempD
 com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds
 
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
+
+@defaultMessage use org.elasticsearch.common.Base64#encodeBytes(byte[])
+org.apache.commons.codec.binary.Base64#encodeBase64(byte[])
+org.apache.commons.codec.binary.Base64#encodeBase64String(byte[])
+
+@defaultMessage use org.elasticsearch.common.Base64#encodeBytes(byte[], int, int, int) with Base64.URL_SAFE
+org.apache.commons.codec.binary.Base64#encodeBase64URLSafe(byte[])
+org.apache.commons.codec.binary.Base64#encodeBase64URLSafeString(byte[])
+
+@defaultMessage use org.elasticsearch.common.Base64#decode
+org.apache.commons.codec.binary.Base64#decodeBase64(java.lang.String)
+org.apache.commons.codec.binary.Base64#decodeBase64(byte[])