add PKI realm

Adds PKI authentication realm that uses X.509 certificates for authentication.
Authorization is provided through the use of role mapping files.

Closes elastic/elasticsearch#782

Original commit: elastic/x-pack-elasticsearch@5a50e50598

src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java

@@ -10,6 +10,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.authc.activedirectory.ActiveDirectoryRealm;
 import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
 import org.elasticsearch.shield.authc.ldap.LdapRealm;
+import org.elasticsearch.shield.authc.pki.PkiRealm;
 import org.elasticsearch.shield.support.AbstractShieldModule;
 
 /**
@@ -27,6 +28,7 @@ public class AuthenticationModule extends AbstractShieldModule.Node {
         mapBinder.addBinding(ESUsersRealm.TYPE).to(ESUsersRealm.Factory.class).asEagerSingleton();
         mapBinder.addBinding(ActiveDirectoryRealm.TYPE).to(ActiveDirectoryRealm.Factory.class).asEagerSingleton();
         mapBinder.addBinding(LdapRealm.TYPE).to(LdapRealm.Factory.class).asEagerSingleton();
+        mapBinder.addBinding(PkiRealm.TYPE).to(PkiRealm.Factory.class).asEagerSingleton();
 
         bind(Realms.class).asEagerSingleton();
         bind(AuthenticationService.class).to(InternalAuthenticationService.class).asEagerSingleton();

src/main/java/org/elasticsearch/shield/authc/RealmConfig.java

@@ -57,6 +57,10 @@ public class RealmConfig {
         return settings;
     }
 
+    public Settings globalSettings() {
+        return globalSettings;
+    }
+
     public ESLogger logger(Class clazz) {
         return Loggers.getLogger(clazz, globalSettings);
     }

src/main/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealm.java

@@ -9,9 +9,8 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.shield.ShieldSettingsFilter;
 import org.elasticsearch.shield.authc.RealmConfig;
-import org.elasticsearch.shield.authc.ldap.LdapUserSearchSessionFactory;
 import org.elasticsearch.shield.authc.ldap.support.AbstractLdapRealm;
-import org.elasticsearch.shield.authc.ldap.support.LdapRoleMapper;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.ssl.ClientSSLService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
@@ -24,7 +23,7 @@ public class ActiveDirectoryRealm extends AbstractLdapRealm {
 
     public ActiveDirectoryRealm(RealmConfig config,
                                 ActiveDirectorySessionFactory connectionFactory,
-                                LdapRoleMapper roleMapper) {
+                                DnRoleMapper roleMapper) {
 
         super(TYPE, config, connectionFactory, roleMapper);
     }
@@ -49,7 +48,7 @@ public class ActiveDirectoryRealm extends AbstractLdapRealm {
         @Override
         public ActiveDirectoryRealm create(RealmConfig config) {
             ActiveDirectorySessionFactory connectionFactory = new ActiveDirectorySessionFactory(config, clientSSLService);
-            LdapRoleMapper roleMapper = new LdapRoleMapper(TYPE, config, watcherService, null);
+            DnRoleMapper roleMapper = new DnRoleMapper(TYPE, config, watcherService, null);
             return new ActiveDirectoryRealm(config, connectionFactory, roleMapper);
         }
     }

src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java

@@ -12,7 +12,7 @@ import org.elasticsearch.shield.ShieldSettingsException;
 import org.elasticsearch.shield.ShieldSettingsFilter;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.ldap.support.AbstractLdapRealm;
-import org.elasticsearch.shield.authc.ldap.support.LdapRoleMapper;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.authc.ldap.support.SessionFactory;
 import org.elasticsearch.shield.ssl.ClientSSLService;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -24,7 +24,7 @@ public class LdapRealm extends AbstractLdapRealm {
 
     public static final String TYPE = "ldap";
 
-    public LdapRealm(RealmConfig config, SessionFactory ldap, LdapRoleMapper roleMapper) {
+    public LdapRealm(RealmConfig config, SessionFactory ldap, DnRoleMapper roleMapper) {
         super(TYPE, config, ldap, roleMapper);
     }
 
@@ -48,7 +48,7 @@ public class LdapRealm extends AbstractLdapRealm {
         @Override
         public LdapRealm create(RealmConfig config) {
             SessionFactory sessionFactory = sessionFactory(config, clientSSLService);
-            LdapRoleMapper roleMapper = new LdapRoleMapper(TYPE, config, watcherService, null);
+            DnRoleMapper roleMapper = new DnRoleMapper(TYPE, config, watcherService, null);
             return new LdapRealm(config, sessionFactory, roleMapper);
         }
 

src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java

@@ -8,10 +8,7 @@ package org.elasticsearch.shield.authc.ldap.support;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.authc.RealmConfig;
-import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
+import org.elasticsearch.shield.authc.support.*;
-import org.elasticsearch.shield.authc.support.RefreshListener;
-import org.elasticsearch.shield.authc.support.UsernamePasswordRealm;
-import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 
 import java.util.List;
 import java.util.Set;
@@ -22,10 +19,10 @@ import java.util.Set;
 public abstract class AbstractLdapRealm extends CachingUsernamePasswordRealm {
 
     protected final SessionFactory sessionFactory;
-    protected final LdapRoleMapper roleMapper;
+    protected final DnRoleMapper roleMapper;
 
     protected AbstractLdapRealm(String type, RealmConfig config,
-                                SessionFactory sessionFactory, LdapRoleMapper roleMapper) {
+                                SessionFactory sessionFactory, DnRoleMapper roleMapper) {
         super(type, config);
         this.sessionFactory = sessionFactory;
         this.roleMapper = roleMapper;

src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java

@@ -0,0 +1,234 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.shield.ShieldSettingsException;
+import org.elasticsearch.shield.ShieldSettingsFilter;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.AuthenticationToken;
+import org.elasticsearch.shield.authc.Realm;
+import org.elasticsearch.shield.authc.RealmConfig;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
+import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
+import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.watcher.ResourceWatcherService;
+
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+import java.io.FileInputStream;
+import java.security.KeyStore;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PkiRealm extends Realm<X509AuthenticationToken> {
+
+    public static final String PKI_CERT_HEADER_NAME = "__SHIELD_CLIENT_CERTIFICATE";
+    public static final String TYPE = "pki";
+
+    // For client based cert validation, the auth type must be specified but UNKNOWN is an acceptable value
+    public static final String AUTH_TYPE = "UNKNOWN";
+
+    private final X509TrustManager[] trustManagers;
+    private final Pattern principalPattern;
+    private final DnRoleMapper roleMapper;
+
+    public PkiRealm(RealmConfig config, DnRoleMapper roleMapper) {
+        super(TYPE, config);
+        this.trustManagers = trustManagers(config.settings());
+        this.principalPattern = Pattern.compile(config.settings().get("username_pattern", "CN=(.*?),"), Pattern.CASE_INSENSITIVE);
+        this.roleMapper = roleMapper;
+        checkSSLEnabled(config, logger);
+    }
+
+    @Override
+    public boolean supports(AuthenticationToken token) {
+        return token instanceof X509AuthenticationToken;
+    }
+
+    @Override
+    public X509AuthenticationToken token(RestRequest request) {
+        return token(request.getFromContext(PKI_CERT_HEADER_NAME), principalPattern, logger);
+    }
+
+    @Override
+    public X509AuthenticationToken token(TransportMessage<?> message) {
+        return token(message.getFromContext(PKI_CERT_HEADER_NAME), principalPattern, logger);
+    }
+
+    @Override
+    public User authenticate(X509AuthenticationToken token) {
+        if (!isCertificateChainTrusted(trustManagers, token, logger)) {
+            return null;
+        }
+
+        Set<String> roles = roleMapper.resolveRoles(token.dn(), Collections.<String>emptyList());
+        return new User.Simple(token.principal(), roles.toArray(new String[roles.size()]));
+    }
+
+    static X509AuthenticationToken token(Object pkiHeaderValue, Pattern principalPattern, ESLogger logger) {
+        if (pkiHeaderValue == null) {
+            return null;
+        }
+
+        assert pkiHeaderValue instanceof X509Certificate[];
+        X509Certificate[] certificates = (X509Certificate[]) pkiHeaderValue;
+        if (certificates.length == 0) {
+            return null;
+        }
+
+        String dn = certificates[0].getSubjectX500Principal().getName();
+        Matcher matcher = principalPattern.matcher(dn);
+        if (!matcher.find()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("certificate authentication succeeded for [{}] but could not extract principal from DN", dn);
+            }
+            return null;
+        }
+
+        String principal = matcher.group(1);
+        return new X509AuthenticationToken(certificates, principal, dn);
+    }
+
+    static boolean isCertificateChainTrusted(X509TrustManager[] trustManagers, X509AuthenticationToken token, ESLogger logger) {
+        if (trustManagers.length > 0) {
+            boolean trusted = false;
+            for (X509TrustManager trustManager : trustManagers) {
+                try {
+                    trustManager.checkClientTrusted(token.credentials(), AUTH_TYPE);
+                    trusted = true;
+                    break;
+                } catch (CertificateException e) {
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("failed certificate validation for principal [{}]", e, token.principal());
+                    } else if (logger.isDebugEnabled()) {
+                        logger.debug("failed certificate validation for principal [{}]", token.principal());
+                    }
+                }
+            }
+
+            return trusted;
+        }
+
+        // No extra trust managers specified, so at this point we can be considered authenticated.
+        return true;
+    }
+
+    static X509TrustManager[] trustManagers(Settings settings) {
+        String truststorePath = settings.get("truststore.path");
+        if (truststorePath == null) {
+            return new X509TrustManager[0];
+        }
+
+        String password = settings.get("truststore.password");
+        if (password == null) {
+            throw new ShieldSettingsException("no truststore password configured");
+        }
+
+        String trustStoreAlgorithm = settings.get("truststore.algorithm", System.getProperty("ssl.TrustManagerFactory.algorithm", TrustManagerFactory.getDefaultAlgorithm()));
+        TrustManager[] trustManagers;
+        try (FileInputStream in = new FileInputStream(truststorePath)) {
+            // Load TrustStore
+            KeyStore ks = KeyStore.getInstance("jks");
+            ks.load(in, password.toCharArray());
+
+            // Initialize a trust manager factory with the trusted store
+            TrustManagerFactory trustFactory = TrustManagerFactory.getInstance(trustStoreAlgorithm);
+            trustFactory.init(ks);
+            trustManagers = trustFactory.getTrustManagers();
+        } catch (Exception e) {
+            throw new ShieldSettingsException("failed to load specified truststore", e);
+        }
+
+        List<X509TrustManager> trustManagerList = new ArrayList<>();
+        for (TrustManager trustManager : trustManagers) {
+            if (trustManager instanceof X509TrustManager) {
+                trustManagerList.add((X509TrustManager) trustManager);
+            }
+        }
+
+        if (trustManagerList.isEmpty()) {
+            throw new ShieldSettingsException("no valid certificates found in truststore");
+        }
+
+        return trustManagerList.toArray(new X509TrustManager[trustManagerList.size()]);
+    }
+
+    static void filterOutSensitiveSettings(String realmName, ShieldSettingsFilter filter) {
+        filter.filterOut("shield.authc.realms." + realmName + "." + "truststore.password");
+        filter.filterOut("shield.authc.realms." + realmName + "." + "truststore.path");
+        filter.filterOut("shield.authc.realms." + realmName + "." + "truststore.algorithm");
+    }
+
+    /**
+     * Checks to see if both SSL and Client authentication are enabled on at least one network communication layer. If
+     * not an error message will be logged
+     * @param config
+     */
+    static void checkSSLEnabled(RealmConfig config, ESLogger logger) {
+        Settings settings = config.globalSettings();
+
+        // HTTP
+        if (settings.getAsBoolean(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, ShieldNettyHttpServerTransport.HTTP_SSL_DEFAULT
+                && settings.getAsBoolean(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING, ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_DEFAULT))) {
+            return;
+        }
+
+        // Default Transport
+        final boolean ssl = settings.getAsBoolean(ShieldNettyTransport.TRANSPORT_SSL_SETTING, ShieldNettyTransport.TRANSPORT_SSL_DEFAULT);
+        final boolean clientAuth = settings.getAsBoolean(ShieldNettyTransport.TRANSPORT_CLIENT_AUTH_SETTING, ShieldNettyTransport.TRANSPORT_CLIENT_AUTH_DEFAULT);
+        if (ssl && clientAuth) {
+            return;
+        }
+
+        // Transport Profiles
+        Map<String, Settings> groupedSettings = settings.getGroups("transport.profiles.");
+        for (Map.Entry<String, Settings> entry : groupedSettings.entrySet()) {
+            Settings profileSettings = entry.getValue().getByPrefix("shield.filter.");
+            if (profileSettings.getAsBoolean(ShieldNettyTransport.TRANSPORT_PROFILE_SSL_SETTING, ssl)
+                    && profileSettings.getAsBoolean(ShieldNettyTransport.TRANSPORT_CLIENT_AUTH_SETTING, clientAuth)) {
+                return;
+            }
+        }
+
+        logger.error("PKI realm [{}] is enabled but cannot be used as neither HTTP or Transport have both SSL and client authentication enabled", config.name());
+    }
+
+    public static class Factory extends Realm.Factory<PkiRealm> {
+
+        private final ResourceWatcherService watcherService;
+
+        @Inject
+        public Factory(ResourceWatcherService watcherService) {
+            super(TYPE, false);
+            this.watcherService = watcherService;
+        }
+
+        @Override
+        public void filterOutSensitiveSettings(String realmName, ShieldSettingsFilter filter) {
+            PkiRealm.filterOutSensitiveSettings(realmName, filter);
+        }
+
+        @Override
+        public PkiRealm create(RealmConfig config) {
+            DnRoleMapper roleMapper = new DnRoleMapper(TYPE, config, watcherService, null);
+            return new PkiRealm(config, roleMapper);
+        }
+
+        @Override
+        public PkiRealm createDefault(String name) {
+            return null;
+        }
+    }
+}

src/main/java/org/elasticsearch/shield/authc/pki/X509AuthenticationToken.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+import org.elasticsearch.shield.authc.AuthenticationToken;
+
+import java.security.cert.X509Certificate;
+
+public class X509AuthenticationToken implements AuthenticationToken {
+
+    private final String principal;
+    private final String dn;
+    private X509Certificate[] credentials;
+
+    public X509AuthenticationToken(X509Certificate[] certificates, String principal, String dn) {
+        this.principal = principal;
+        this.credentials = certificates;
+        this.dn = dn;
+    }
+
+    @Override
+    public String principal() {
+        return principal;
+    }
+
+    @Override
+    public X509Certificate[] credentials() {
+        return credentials;
+    }
+
+    public String dn() {
+        return dn;
+    }
+
+    @Override
+    public void clearCredentials() {
+        credentials = null;
+    }
+}

src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.authc.ldap.support;
+package org.elasticsearch.shield.authc.support;
 
 import com.unboundid.ldap.sdk.DN;
 import com.unboundid.ldap.sdk.LDAPException;
@@ -16,7 +16,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.ShieldSettingsException;
 import org.elasticsearch.shield.authc.RealmConfig;
-import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -34,9 +33,9 @@ import static org.elasticsearch.shield.authc.ldap.support.LdapUtils.dn;
 import static org.elasticsearch.shield.authc.ldap.support.LdapUtils.relativeName;
 
 /**
- * This class loads and monitors the file defining the mappings of LDAP DNs to internal ES Roles.
+ * This class loads and monitors the file defining the mappings of DNs to internal ES Roles.
  */
-public class LdapRoleMapper {
+public class DnRoleMapper {
 
     public static final String DEFAULT_FILE_NAME = "role_mapping.yml";
     public static final String ROLE_MAPPING_FILE_SETTING = "files.role_mapping";
@@ -52,7 +51,7 @@ public class LdapRoleMapper {
 
     private CopyOnWriteArrayList<RefreshListener> listeners;
 
-    public LdapRoleMapper(String realmType, RealmConfig config, ResourceWatcherService watcherService, @Nullable RefreshListener listener) {
+    public DnRoleMapper(String realmType, RealmConfig config, ResourceWatcherService watcherService, @Nullable RefreshListener listener) {
         this.realmType = realmType;
         this.config = config;
         this.logger = config.logger(getClass());
@@ -111,9 +110,9 @@ public class LdapRoleMapper {
             Map<DN, Set<String>> dnToRoles = new HashMap<>();
             Set<String> roles = settings.names();
             for (String role : roles) {
-                for (String ldapDN : settings.getAsArray(role)) {
+                for (String providedDn : settings.getAsArray(role)) {
                     try {
-                        DN dn = new DN(ldapDN);
+                        DN dn = new DN(providedDn);
                         Set<String> dnRoles = dnToRoles.get(dn);
                         if (dnRoles == null) {
                             dnRoles = new HashSet<>();
@@ -121,7 +120,7 @@ public class LdapRoleMapper {
                         }
                         dnRoles.add(role);
                     } catch (LDAPException e) {
-                        logger.error("invalid DN [{}] found in [{}] role mappings [{}] for realm [{}/{}]. skipping... ", e, ldapDN, realmType, path.toAbsolutePath(), realmType, realmName);
+                        logger.error("invalid DN [{}] found in [{}] role mappings [{}] for realm [{}/{}]. skipping... ", e, providedDn, realmType, path.toAbsolutePath(), realmType, realmName);
                     }
                 }
 
@@ -147,12 +146,12 @@ public class LdapRoleMapper {
      */
     public Set<String> resolveRoles(String userDnString, List<String> groupDns) {
         Set<String> roles = new HashSet<>();
-        for (String groupDn : groupDns) {
+        for (String groupDnString : groupDns) {
-            DN groupLdapName = dn(groupDn);
+            DN groupDn = dn(groupDnString);
-            if (dnRoles.containsKey(groupLdapName)) {
+            if (dnRoles.containsKey(groupDn)) {
-                roles.addAll(dnRoles.get(groupLdapName));
+                roles.addAll(dnRoles.get(groupDn));
             } else if (useUnmappedGroupsAsRoles) {
-                roles.add(relativeName(groupLdapName));
+                roles.add(relativeName(groupDn));
             }
         }
         if (logger.isDebugEnabled()) {
@@ -189,7 +188,7 @@ public class LdapRoleMapper {
 
         @Override
         public void onFileChanged(File file) {
-            if (file.equals(LdapRoleMapper.this.file.toFile())) {
+            if (file.equals(DnRoleMapper.this.file.toFile())) {
                 logger.info("role mappings file [{}] changed for realm [{}/{}]. updating mappings...", file.getAbsolutePath(), realmType, config.name());
                 dnRoles = parseFileLenient(file.toPath(), logger, realmType, config.name());
                 notifyRefresh();

src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java

@@ -6,8 +6,16 @@
 package org.elasticsearch.shield.rest;
 
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.netty.handler.ssl.SslHandler;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.netty.NettyHttpRequest;
 import org.elasticsearch.rest.*;
 import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.shield.authc.pki.PkiRealm;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
+
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
 
 /**
  *
@@ -15,11 +23,15 @@ import org.elasticsearch.shield.authc.AuthenticationService;
 public class ShieldRestFilter extends RestFilter {
 
     private final AuthenticationService service;
+    private final boolean extractClientCertificate;
 
     @Inject
-    public ShieldRestFilter(AuthenticationService service, RestController controller) {
+    public ShieldRestFilter(AuthenticationService service, RestController controller, Settings settings) {
         this.service = service;
         controller.registerFilter(this);
+        boolean useClientAuth = settings.getAsBoolean(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING, ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_DEFAULT);
+        boolean ssl = settings.getAsBoolean(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, ShieldNettyHttpServerTransport.HTTP_SSL_DEFAULT);
+        extractClientCertificate = ssl && useClientAuth;
     }
 
     @Override
@@ -32,6 +44,9 @@ public class ShieldRestFilter extends RestFilter {
 
         // CORS - allow for preflight unauthenticated OPTIONS request
         if (request.method() != RestRequest.Method.OPTIONS) {
+            if (extractClientCertificate) {
+                putClientCertificateInContext(request);
+            }
             service.authenticate(request);
         }
 
@@ -39,4 +54,16 @@ public class ShieldRestFilter extends RestFilter {
 
         filterChain.continueProcessing(request, channel);
     }
+
+    static void putClientCertificateInContext(RestRequest request) throws Exception {
+        assert request instanceof NettyHttpRequest;
+        NettyHttpRequest nettyHttpRequest = (NettyHttpRequest) request;
+
+        SslHandler handler = nettyHttpRequest.getChannel().getPipeline().get(SslHandler.class);
+        assert handler != null;
+        Certificate[] certs = handler.getEngine().getSession().getPeerCertificates();
+        if (certs instanceof X509Certificate[]) {
+            request.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, certs);
+        }
+    }
 }

src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyHttpServerTransport.java

@@ -26,6 +26,11 @@ import javax.net.ssl.SSLEngine;
  */
 public class ShieldNettyHttpServerTransport extends NettyHttpServerTransport {
 
+    public static final String HTTP_SSL_SETTING = "shield.http.ssl";
+    public static final boolean HTTP_SSL_DEFAULT = false;
+    public static final String HTTP_CLIENT_AUTH_SETTING = "shield.http.ssl.client.auth";
+    public static final boolean HTTP_CLIENT_AUTH_DEFAULT = false;
+
     private final IPFilter ipFilter;
     private final ServerSSLService sslService;
     private final boolean ssl;
@@ -35,7 +40,7 @@ public class ShieldNettyHttpServerTransport extends NettyHttpServerTransport {
                                           IPFilter ipFilter, ServerSSLService sslService) {
         super(settings, networkService, bigArrays);
         this.ipFilter = ipFilter;
-        this.ssl = settings.getAsBoolean("shield.http.ssl", false);
+        this.ssl = settings.getAsBoolean(HTTP_SSL_SETTING, HTTP_SSL_DEFAULT);
         this.sslService =  sslService;
     }
 
@@ -60,8 +65,11 @@ public class ShieldNettyHttpServerTransport extends NettyHttpServerTransport {
 
     private class HttpSslChannelPipelineFactory extends HttpChannelPipelineFactory {
 
+        private final boolean useClientAuth;
+
         public HttpSslChannelPipelineFactory(NettyHttpServerTransport transport) {
             super(transport, detailedErrorsEnabled);
+            useClientAuth = settings.getAsBoolean(HTTP_CLIENT_AUTH_SETTING, HTTP_CLIENT_AUTH_DEFAULT);
         }
 
         @Override
@@ -70,7 +78,7 @@ public class ShieldNettyHttpServerTransport extends NettyHttpServerTransport {
             if (ssl) {
                 SSLEngine engine = sslService.createSSLEngine();
                 engine.setUseClientMode(false);
-                engine.setNeedClientAuth(settings.getAsBoolean("shield.http.ssl.client.auth", false));
+                engine.setNeedClientAuth(useClientAuth);
 
                 pipeline.addFirst("ssl", new SslHandler(engine));
             }

src/main/java/org/elasticsearch/shield/transport/netty/ShieldNettyTransport.java

@@ -20,6 +20,7 @@ import org.elasticsearch.shield.ssl.ServerSSLService;
 import org.elasticsearch.shield.transport.filter.IPFilter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty.NettyTransport;
+import org.elasticsearch.transport.netty.ShieldMessageChannelHandler;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
@@ -32,6 +33,12 @@ public class ShieldNettyTransport extends NettyTransport {
 
     public static final String HOSTNAME_VERIFICATION_SETTING = "shield.ssl.hostname_verification";
     public static final String HOSTNAME_VERIFICATION_RESOLVE_NAME_SETTING = "shield.ssl.hostname_verification.resolve_name";
+    public static final String TRANSPORT_SSL_SETTING = "shield.transport.ssl";
+    public static final boolean TRANSPORT_SSL_DEFAULT = false;
+    public static final String TRANSPORT_CLIENT_AUTH_SETTING = "shield.transport.ssl.client.auth";
+    public static final boolean TRANSPORT_CLIENT_AUTH_DEFAULT = true;
+    public static final String TRANSPORT_PROFILE_SSL_SETTING = "shield.ssl";
+    public static final String TRANSPORT_PROFILE_CLIENT_AUTH_SETTING = "shield.ssl.client.auth";
 
     private final ServerSSLService serverSslService;
     private final ClientSSLService clientSSLService;
@@ -45,7 +52,7 @@ public class ShieldNettyTransport extends NettyTransport {
                                 ShieldSettingsFilter settingsFilter) {
         super(settings, threadPool, networkService, bigArrays, version);
         this.authenticator = authenticator;
-        this.ssl = settings.getAsBoolean("shield.transport.ssl", false);
+        this.ssl = settings.getAsBoolean(TRANSPORT_SSL_SETTING, TRANSPORT_SSL_DEFAULT);
         this.serverSslService = serverSSLService;
         this.clientSSLService = clientSSLService;
         this.settingsFilter = settingsFilter;
@@ -74,7 +81,8 @@ public class ShieldNettyTransport extends NettyTransport {
         @Override
         public ChannelPipeline getPipeline() throws Exception {
             ChannelPipeline pipeline = super.getPipeline();
-            boolean profileSsl = profileSettings.getAsBoolean("shield.ssl", ssl);
+            final boolean profileSsl = profileSettings.getAsBoolean(TRANSPORT_PROFILE_SSL_SETTING, ssl);
+            final boolean needClientAuth = profileSettings.getAsBoolean(TRANSPORT_PROFILE_CLIENT_AUTH_SETTING, settings.getAsBoolean(TRANSPORT_CLIENT_AUTH_SETTING, TRANSPORT_CLIENT_AUTH_DEFAULT));
             if (profileSsl) {
                 SSLEngine serverEngine;
                 if (profileSettings.get("shield.truststore.path") != null) {
@@ -83,13 +91,15 @@ public class ShieldNettyTransport extends NettyTransport {
                     serverEngine = serverSslService.createSSLEngine();
                 }
                 serverEngine.setUseClientMode(false);
-                serverEngine.setNeedClientAuth(profileSettings.getAsBoolean("shield.ssl.client.auth", settings.getAsBoolean("shield.transport.ssl.client.auth", true)));
+                serverEngine.setNeedClientAuth(needClientAuth);
 
                 pipeline.addFirst("ssl", new SslHandler(serverEngine));
             }
             if (authenticator != null) {
                 pipeline.addFirst("ipfilter", new IPFilterNettyUpstreamHandler(authenticator, name));
             }
+            boolean extractClientCert = profileSsl && needClientAuth;
+            pipeline.replace("dispatcher", "dispatcher", new ShieldMessageChannelHandler(ShieldNettyTransport.this, logger, name, extractClientCert));
             return pipeline;
         }
     }

src/main/java/org/elasticsearch/transport/netty/ShieldMessageChannelHandler.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.transport.netty;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.netty.channel.Channel;
+import org.elasticsearch.common.netty.handler.ssl.SslHandler;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.shield.authc.pki.PkiRealm;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ActionNotFoundTransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+
+// TODO remove this class and add ability for transport filters to access netty channels
+public class ShieldMessageChannelHandler extends MessageChannelHandler {
+
+    protected final boolean extractClientCert;
+
+    public ShieldMessageChannelHandler(NettyTransport transport, ESLogger logger, String profileName, boolean extractClientCert) {
+        super(transport, logger, profileName);
+        this.extractClientCert = extractClientCert;
+    }
+
+    protected String handleRequest(Channel channel, StreamInput buffer, long requestId, Version version) throws IOException {
+        final String action = buffer.readString();
+        transportServiceAdapter.onRequestReceived(requestId, action);
+        final NettyTransportChannel transportChannel = new NettyTransportChannel(transport, transportServiceAdapter, action, channel, requestId, version, profileName);
+        try {
+            final TransportRequestHandler handler = transportServiceAdapter.handler(action, version);
+            if (handler == null) {
+                throw new ActionNotFoundTransportException(action);
+            }
+            final TransportRequest request = handler.newInstance();
+            request.remoteAddress(new InetSocketTransportAddress((InetSocketAddress) channel.getRemoteAddress()));
+            request.readFrom(buffer);
+
+            if (extractClientCert) {
+                SslHandler sslHandler = channel.getPipeline().get(SslHandler.class);
+                assert sslHandler != null;
+                Certificate[] certs = sslHandler.getEngine().getSession().getPeerCertificates();
+                if (certs instanceof X509Certificate[]) {
+                    request.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, certs);
+                }
+            }
+
+            if (ThreadPool.Names.SAME.equals(handler.executor())) {
+                //noinspection unchecked
+                handler.messageReceived(request, transportChannel);
+            } else {
+                threadPool.executor(handler.executor()).execute(new RequestHandler(handler, request, transportChannel, action));
+            }
+        } catch (Throwable e) {
+            try {
+                transportChannel.sendResponse(e);
+            } catch (IOException e1) {
+                logger.warn("Failed to send error message back to client for action [" + action + "]", e);
+                logger.warn("Actual Exception", e1);
+            }
+        }
+        return action;
+    }
+}

src/test/java/org/elasticsearch/integration/SettingsFilterTests.java

@@ -9,7 +9,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -23,7 +22,6 @@ import org.elasticsearch.test.ShieldSettingsSource;
 import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
 import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -70,6 +68,13 @@ public class SettingsFilterTests extends ShieldIntegrationTest {
                 .put("shield.authc.realms.ad1.url", "ldap://host.domain")
                 .put("shield.authc.realms.ad1.hostname_verification", randomAsciiOfLength(5))
 
+                // pki filtering
+                .put("shield.authc.realms.pki1.type", "pki")
+                .put("shield.authc.realms.pki1.order", "0")
+                .put("shield.authc.realms.pki1.truststore.path", getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks"))
+                .put("shield.authc.realms.pki1.truststore.password", "truststore-testnode-only")
+                .put("shield.authc.realms.pki1.truststore.algorithm", "SunX509")
+
                 .put("shield.ssl.keystore.path", "/path/to/keystore")
                 .put("shield.ssl.ciphers", "_ciphers")
                 .put("shield.ssl.supported_protocols", randomFrom(AbstractSSLService.DEFAULT_SUPPORTED_PROTOCOLS))
@@ -118,6 +123,11 @@ public class SettingsFilterTests extends ShieldIntegrationTest {
             assertThat(settings.get("shield.authc.realms.ad1.hostname_verification"), nullValue());
             assertThat(settings.get("shield.authc.realms.ad1.url"), is("ldap://host.domain"));
 
+            assertThat(settings.get("shield.authc.realms.pki1.truststore.path"), nullValue());
+            assertThat(settings.get("shield.authc.realms.pki1.truststore.password"), nullValue());
+            assertThat(settings.get("shield.authc.realms.pki1.truststore.algorithm"), nullValue());
+            assertThat(settings.get("shield.authc.realms.pki1.type"), is("pki"));
+
             assertThat(settings.get("shield.ssl.keystore.path"), nullValue());
             assertThat(settings.get("shield.ssl.ciphers"), nullValue());
             assertThat(settings.get("shield.ssl.supported_protocols"), nullValue());

src/test/java/org/elasticsearch/rest/FakeRestRequest.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.rest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// TODO fix in core so it is packaged with test jar and remove
+public class FakeRestRequest extends RestRequest {
+
+    private final Map<String, String> headers;
+
+    private final Map<String, String> params;
+
+    public FakeRestRequest() {
+        this(new HashMap<String, String>(), new HashMap<String, String>());
+    }
+
+    public FakeRestRequest(Map<String, String> headers, Map<String, String> context) {
+        this.headers = headers;
+        for (Map.Entry<String, String> entry : context.entrySet()) {
+            putInContext(entry.getKey(), entry.getValue());
+        }
+        this.params = new HashMap<>();
+    }
+
+    @Override
+    public Method method() {
+        return Method.GET;
+    }
+
+    @Override
+    public String uri() {
+        return "/";
+    }
+
+    @Override
+    public String rawPath() {
+        return "/";
+    }
+
+    @Override
+    public boolean hasContent() {
+        return false;
+    }
+
+    @Override
+    public boolean contentUnsafe() {
+        return false;
+    }
+
+    @Override
+    public BytesReference content() {
+        return null;
+    }
+
+    @Override
+    public String header(String name) {
+        return headers.get(name);
+    }
+
+    @Override
+    public Iterable<Map.Entry<String, String>> headers() {
+        return headers.entrySet();
+    }
+
+    @Override
+    public boolean hasParam(String key) {
+        return params.containsKey(key);
+    }
+
+    @Override
+    public String param(String key) {
+        return params.get(key);
+    }
+
+    @Override
+    public String param(String key, String defaultValue) {
+        String value = params.get(key);
+        if (value == null) {
+            return defaultValue;
+        }
+        return value;
+    }
+
+    @Override
+    public Map<String, String> params() {
+        return params;
+    }
+}

src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java

@@ -33,5 +33,19 @@ public class VersionCompatibilityTests extends ElasticsearchTestCase {
          *
          */
         assertThat("Remove workaround in LicenseService class when es core supports merging cluster level custom metadata", Version.CURRENT.onOrBefore(Version.V_1_5_0), is(true));
+
+        /**
+         * see https://github.com/elastic/elasticsearch/pull/10319 {@link org.elasticsearch.transport.netty.ShieldMessageChannelHandler}
+         * Once ES core supports exposing the channel in {@link org.elasticsearch.transport.netty.NettyTransportChannel}
+         * we should implement the certificate extraction logic as a {@link org.elasticsearch.shield.transport.ServerTransportFilter}
+         */
+        assertThat("Remove ShieldMessageChannelHandler and implement PKI cert extraction as a ServerTransportFilter", Version.CURRENT.onOrBefore(Version.V_1_5_0), is(true));
+
+        /**
+         * see https://github.com/elastic/elasticsearch/pull/10323 {@link org.elasticsearch.rest.FakeRestRequest}
+         * ES core has FakeRestRequest but it is not included in the test jar. Once it is included in the test jar, Shield
+         * should be updated to remove the copied version of the class {@link org.elasticsearch.rest.FakeRestRequest}
+         */
+        assertThat("Remove FakeRestRequest and use version in core", Version.CURRENT.onOrBefore(Version.V_1_5_0), is(true));
     }
 }

src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java

@@ -13,10 +13,9 @@ import com.unboundid.ldap.sdk.LDAPURL;
 import com.unboundid.ldap.sdk.schema.Schema;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.authc.RealmConfig;
-import org.elasticsearch.shield.authc.ldap.support.LdapRoleMapper;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
@@ -95,7 +94,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         Settings settings = settings();
         RealmConfig config = new RealmConfig("testAuthenticateUserPrincipleName", settings);
         ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, null);
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         User user = realm.authenticate(new UsernamePasswordToken("CN=ironman", SecuredStringTests.build(PASSWORD)));
@@ -108,7 +107,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         Settings settings = settings();
         RealmConfig config = new RealmConfig("testAuthenticateSAMAccountName", settings);
         ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, null);
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         // Thor does not have a UPN of form CN=Thor@ad.test.elasticsearch.com
@@ -127,7 +126,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         Settings settings = settings();
         RealmConfig config = new RealmConfig("testAuthenticateCachesSuccesfulAuthentications", settings);
         ActiveDirectorySessionFactory sessionFactory = spy(new ActiveDirectorySessionFactory(config, null));
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         int count = randomIntBetween(2, 10);
@@ -144,7 +143,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         Settings settings = settings(ImmutableSettings.builder().put(CachingUsernamePasswordRealm.CACHE_TTL_SETTING, -1).build());
         RealmConfig config = new RealmConfig("testAuthenticateCachingCanBeDisabled", settings);
         ActiveDirectorySessionFactory sessionFactory = spy(new ActiveDirectorySessionFactory(config, null));
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         int count = randomIntBetween(2, 10);
@@ -161,7 +160,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         Settings settings = settings();
         RealmConfig config = new RealmConfig("testAuthenticateCachingClearsCacheOnRoleMapperRefresh", settings);
         ActiveDirectorySessionFactory sessionFactory = spy(new ActiveDirectorySessionFactory(config, null));
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         int count = randomIntBetween(2, 10);
@@ -185,11 +184,11 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
     @Test
     public void testRealmMapsGroupsToRoles() throws Exception {
         Settings settings = settings(ImmutableSettings.builder()
-                .put(LdapRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("role_mapping.yml").getCanonicalPath())
+                .put(DnRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("role_mapping.yml").getCanonicalPath())
                 .build());
         RealmConfig config = new RealmConfig("testRealmMapsGroupsToRoles", settings);
         ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, null);
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         User user = realm.authenticate(new UsernamePasswordToken("CN=ironman", SecuredStringTests.build(PASSWORD)));
@@ -200,11 +199,11 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
     @Test
     public void testRealmMapsUsersToRoles() throws Exception {
         Settings settings = settings(ImmutableSettings.builder()
-                .put(LdapRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("role_mapping.yml").getCanonicalPath())
+                .put(DnRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("role_mapping.yml").getCanonicalPath())
                 .build());
         RealmConfig config = new RealmConfig("testRealmMapsGroupsToRoles", settings);
         ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, null);
-        LdapRoleMapper roleMapper = new LdapRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
+        DnRoleMapper roleMapper = new DnRoleMapper(ActiveDirectoryRealm.TYPE, config, resourceWatcherService, null);
         ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, roleMapper);
 
         User user = realm.authenticate(new UsernamePasswordToken("CN=Thor", SecuredStringTests.build(PASSWORD)));
@@ -220,7 +219,7 @@ public class ActiveDirectoryRealmTests extends ElasticsearchTestCase {
         return ImmutableSettings.builder()
                 .putArray(URLS_SETTING, ldapUrl())
                 .put(ActiveDirectorySessionFactory.AD_DOMAIN_NAME_SETTING, "ad.test.elasticsearch.com")
-                .put(LdapRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
+                .put(DnRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
                 .put(HOSTNAME_VERIFICATION_SETTING, false)
                 .put(extraSettings)
                 .build();

src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java

@@ -15,7 +15,7 @@ import org.elasticsearch.shield.authc.ldap.support.SessionFactory;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
-import org.elasticsearch.shield.authc.ldap.support.LdapRoleMapper;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.authc.ldap.support.LdapTest;
 import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -121,7 +121,7 @@ public class LdapRealmTests extends LdapTest {
         RealmConfig config = new RealmConfig("test-ldap-realm", settings);
 
         LdapSessionFactory ldapFactory = new LdapSessionFactory(config, null);
-        LdapRoleMapper roleMapper = buildGroupAsRoleMapper(resourceWatcherService);
+        DnRoleMapper roleMapper = buildGroupAsRoleMapper(resourceWatcherService);
         ldapFactory = spy(ldapFactory);
         LdapRealm ldap = new LdapRealm(config, ldapFactory, roleMapper);
         ldap.authenticate( new UsernamePasswordToken(VALID_USERNAME, SecuredStringTests.build(PASSWORD)));
@@ -220,12 +220,12 @@ public class LdapRealmTests extends LdapTest {
         String userTemplate = VALID_USER_TEMPLATE;
         Settings settings = ImmutableSettings.builder()
                 .put(buildLdapSettings(ldapUrl(), userTemplate, groupSearchBase, LdapSearchScope.SUB_TREE))
-                .put(LdapRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("support/role_mapping.yml").getCanonicalPath())
+                .put(DnRoleMapper.ROLE_MAPPING_FILE_SETTING, getResource("/org/elasticsearch/shield/authc/support/role_mapping.yml").getCanonicalPath())
                 .build();
         RealmConfig config = new RealmConfig("test-ldap-realm-userdn", settings);
 
         LdapSessionFactory ldapFactory = new LdapSessionFactory(config, null);
-        LdapRealm ldap = new LdapRealm(config, ldapFactory, new LdapRoleMapper(LdapRealm.TYPE, config, resourceWatcherService, null));
+        LdapRealm ldap = new LdapRealm(config, ldapFactory, new DnRoleMapper(LdapRealm.TYPE, config, resourceWatcherService, null));
 
         User user = ldap.authenticate(new UsernamePasswordToken("Horatio Hornblower", SecuredStringTests.build(PASSWORD)));
         assertThat(user, notNullValue());

src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapTest.java

@@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.ldap.LdapRealm;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.AfterClass;
@@ -65,12 +66,12 @@ public abstract class LdapTest extends ElasticsearchTestCase {
                 .put(HOSTNAME_VERIFICATION_SETTING, hostnameVerification).build();
     }
 
-    protected LdapRoleMapper buildGroupAsRoleMapper(ResourceWatcherService resourceWatcherService) {
+    protected DnRoleMapper buildGroupAsRoleMapper(ResourceWatcherService resourceWatcherService) {
         Settings settings = ImmutableSettings.builder()
-                .put(LdapRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
+                .put(DnRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
                 .build();
         RealmConfig config = new RealmConfig("ldap1", settings);
 
-        return new LdapRoleMapper(LdapRealm.TYPE, config, resourceWatcherService, null);
+        return new DnRoleMapper(LdapRealm.TYPE, config, resourceWatcherService, null);
     }
 }

src/test/java/org/elasticsearch/shield/authc/pki/PkiAuthenticationTests.java

@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.node.internal.InternalNode;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ShieldIntegrationTest;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.transport.Transport;
+import org.junit.Test;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.*;
+
+/**
+ * Test authentication via PKI on both REST and Transport layers
+ */
+@ClusterScope(numClientNodes = 0, numDataNodes = 1)
+public class PkiAuthenticationTests extends ShieldIntegrationTest {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(InternalNode.HTTP_ENABLED, true)
+                .put(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, true)
+                .put(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING, true)
+                .put("shield.authc.realms.pki1.type", "pki")
+                .put("shield.authc.realms.pki1.order", "0")
+                .put("shield.authc.realms.pki1.truststore.path", getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/truststore-testnode-only.jks"))
+                .put("shield.authc.realms.pki1.truststore.password", "truststore-testnode-only")
+                .put("shield.authc.realms.pki1.files.role_mapping", getResource("role_mapping.yml"))
+                .build();
+    }
+
+    @Override
+    protected boolean sslTransportEnabled() {
+        return true;
+    }
+
+    @Test
+    public void testTransportClientCanAuthenticateViaPki() {
+        Settings settings = ShieldSettingsSource.getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks", "testnode");
+        try (TransportClient client = createTransportClient(settings)) {
+            client.addTransportAddress(internalCluster().getInstance(Transport.class).boundAddress().boundAddress());
+            IndexResponse response = client.prepareIndex("foo", "bar").setSource("pki", "auth").get();
+            assertThat(response.isCreated(), is(true));
+        }
+    }
+
+    /**
+     * Test uses the testclient cert which is trusted by the SSL layer BUT it is not trusted by the PKI authentication
+     * realm
+     */
+    @Test(expected = NoNodeAvailableException.class)
+    public void testTransportClientAuthenticationFailure() {
+        try (TransportClient client = createTransportClient(ImmutableSettings.EMPTY)) {
+            client.addTransportAddress(internalCluster().getInstance(Transport.class).boundAddress().boundAddress());
+            client.prepareIndex("foo", "bar").setSource("pki", "auth").get();
+            fail("transport client should not have been able to authenticate");
+        }
+    }
+
+    @Test
+    public void testRestAuthenticationViaPki() throws Exception {
+        SSLContext context = getRestSSLContext("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks", "testnode");
+        try (CloseableHttpClient client = HttpClients.custom().setSslcontext(context).build()) {
+            HttpPut put = new HttpPut(getNodeUrl() + "foo");
+            try (CloseableHttpResponse response = client.execute(put)) {
+                String body = EntityUtils.toString(response.getEntity());
+                assertThat(body, containsString("\"acknowledged\":true"));
+            }
+        }
+    }
+
+    @Test
+    public void testRestAuthenticationFailure() throws Exception {
+        SSLContext context = getRestSSLContext("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient");
+        try (CloseableHttpClient client = HttpClients.custom().setSslcontext(context).build()) {
+            HttpPut put = new HttpPut(getNodeUrl() + "foo");
+            try (CloseableHttpResponse response = client.execute(put)) {
+                assertThat(response.getStatusLine().getStatusCode(), is(401));
+                String body = EntityUtils.toString(response.getEntity());
+                assertThat(body, containsString("AuthenticationException[unable to authenticate user [Elasticsearch Test Client]"));
+            }
+        }
+    }
+
+    private static SSLContext getRestSSLContext(String keystoreResourcePath, String password) throws Exception {
+        SSLContext context = SSLContext.getInstance("TLS");
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        Path store = Paths.get(PkiAuthenticationTests.class.getResource(keystoreResourcePath).toURI());
+        KeyStore ks;
+        try (InputStream in = Files.newInputStream(store)) {
+            ks = KeyStore.getInstance("jks");
+            ks.load(in, password.toCharArray());
+        }
+
+        kmf.init(ks, password.toCharArray());
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(ks);
+        context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
+
+        return context;
+    }
+
+    private TransportClient createTransportClient(Settings additionalSettings) {
+        ImmutableSettings.Builder builder = ImmutableSettings.builder()
+                .put(transportClientSettings())
+                .put(additionalSettings)
+                .put("cluster.name", internalCluster().getClusterName());
+        builder.remove("shield.user");
+        builder.remove("request.headers.Authorization");
+        return new TransportClient(builder.build());
+    }
+
+    private String getNodeUrl() {
+        TransportAddress transportAddress = internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddress();
+        assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
+        InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
+        return String.format(Locale.ROOT, "https://localhost:%s/", inetSocketTransportAddress.address().getPort());
+    }
+}

src/test/java/org/elasticsearch/shield/authc/pki/PkiRealmTests.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.FakeRestRequest;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.shield.ShieldSettingsException;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.RealmConfig;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.support.NoOpLogger;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.elasticsearch.transport.TransportMessage;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.*;
+import static org.mockito.Mockito.*;
+
+public class PkiRealmTests extends ElasticsearchTestCase {
+
+    @Test
+    public void testTokenSupport() {
+        RealmConfig config = new RealmConfig("");
+        PkiRealm realm = new PkiRealm(config, mock(DnRoleMapper.class));
+
+        assertThat(realm.supports(null), is(false));
+        assertThat(realm.supports(new UsernamePasswordToken("", new SecuredString(new char[0]))), is(false));
+        assertThat(realm.supports(new X509AuthenticationToken(new X509Certificate[0], "", "")), is(true));
+    }
+
+    @Test
+    public void extractTokenFromRestRequest() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        RestRequest restRequest = new FakeRestRequest();
+        restRequest.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, new X509Certificate[] { certificate });
+        PkiRealm realm = new PkiRealm(new RealmConfig(""), mock(DnRoleMapper.class));
+
+        X509AuthenticationToken token = realm.token(restRequest);
+        assertThat(token, is(notNullValue()));
+        assertThat(token.dn(), is("CN=Elasticsearch Test Node,OU=elasticsearch,O=org"));
+        assertThat(token.principal(), is("Elasticsearch Test Node"));
+    }
+
+    @Test
+    public void extractTokenFromTransportMessage() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        Message message = new Message();
+        message.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, new X509Certificate[]{certificate});
+        PkiRealm realm = new PkiRealm(new RealmConfig(""), mock(DnRoleMapper.class));
+
+        X509AuthenticationToken token = realm.token(message);
+        assertThat(token, is(notNullValue()));
+        assertThat(token.dn(), is("CN=Elasticsearch Test Node,OU=elasticsearch,O=org"));
+        assertThat(token.principal(), is("Elasticsearch Test Node"));
+    }
+
+    @Test
+    public void authenticateBasedOnCertToken() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        X509AuthenticationToken token = new X509AuthenticationToken(new X509Certificate[] { certificate }, "Elasticsearch Test Node", "CN=Elasticsearch Test Node,");
+        DnRoleMapper roleMapper = mock(DnRoleMapper.class);
+        PkiRealm realm = new PkiRealm(new RealmConfig(""), roleMapper);
+        when(roleMapper.resolveRoles(anyString(), anyList())).thenReturn(Collections.emptySet());
+
+        User user = realm.authenticate(token);
+        assertThat(user, is(notNullValue()));
+        assertThat(user.principal(), is("Elasticsearch Test Node"));
+        assertThat(user.roles(), is(notNullValue()));
+        assertThat(user.roles().length, is(0));
+    }
+
+    @Test
+    public void customUsernamePattern() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        DnRoleMapper roleMapper = mock(DnRoleMapper.class);
+        PkiRealm realm = new PkiRealm(new RealmConfig("", ImmutableSettings.builder().put("username_pattern", "OU=(.*?),").build()), roleMapper);
+        when(roleMapper.resolveRoles(anyString(), anyList())).thenReturn(Collections.emptySet());
+        FakeRestRequest restRequest = new FakeRestRequest();
+        restRequest.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, new X509Certificate[] { certificate });
+
+        X509AuthenticationToken token = realm.token(restRequest);
+        User user = realm.authenticate(token);
+        assertThat(user, is(notNullValue()));
+        assertThat(user.principal(), is("elasticsearch"));
+        assertThat(user.roles(), is(notNullValue()));
+        assertThat(user.roles().length, is(0));
+    }
+
+    @Test
+    public void verificationUsingATruststore() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        DnRoleMapper roleMapper = mock(DnRoleMapper.class);
+        Settings settings = ImmutableSettings.builder()
+                .put("truststore.path", Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks").toURI()).toAbsolutePath())
+                .put("truststore.password", "testnode")
+                .build();
+        PkiRealm realm = new PkiRealm(new RealmConfig("", settings), roleMapper);
+        when(roleMapper.resolveRoles(anyString(), anyList())).thenReturn(Collections.emptySet());
+
+        FakeRestRequest restRequest = new FakeRestRequest();
+        restRequest.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, new X509Certificate[] { certificate });
+
+        X509AuthenticationToken token = realm.token(restRequest);
+        User user = realm.authenticate(token);
+        assertThat(user, is(notNullValue()));
+        assertThat(user.principal(), is("Elasticsearch Test Node"));
+        assertThat(user.roles(), is(notNullValue()));
+        assertThat(user.roles().length, is(0));
+    }
+
+    @Test
+    public void verificationFailsUsingADifferentTruststore() throws Exception {
+        X509Certificate certificate = readCert(Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.cert").toURI()));
+        DnRoleMapper roleMapper = mock(DnRoleMapper.class);
+        Settings settings = ImmutableSettings.builder()
+                .put("truststore.path", Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks").toURI()).toAbsolutePath())
+                .put("truststore.password", "testnode-client-profile")
+                .build();
+        PkiRealm realm = new PkiRealm(new RealmConfig("", settings), roleMapper);
+        when(roleMapper.resolveRoles(anyString(), anyList())).thenReturn(Collections.emptySet());
+
+        FakeRestRequest restRequest = new FakeRestRequest();
+        restRequest.putInContext(PkiRealm.PKI_CERT_HEADER_NAME, new X509Certificate[] { certificate });
+
+        X509AuthenticationToken token = realm.token(restRequest);
+        User user = realm.authenticate(token);
+        assertThat(user, is(nullValue()));
+    }
+
+    @Test
+    public void truststorePathWithoutPasswordThrowsException() throws Exception {
+        Settings settings = ImmutableSettings.builder()
+                .put("truststore.path", Paths.get(PkiRealmTests.class.getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode-client-profile.jks").toURI()).toAbsolutePath())
+                .build();
+        try {
+            new PkiRealm(new RealmConfig("", settings), mock(DnRoleMapper.class));
+            fail("exception should have been thrown");
+        } catch (ShieldSettingsException e) {
+            assertThat(e.getMessage(), containsString("no truststore password configured"));
+        }
+    }
+
+    static X509Certificate readCert(Path path) throws Exception {
+        try (InputStream in = Files.newInputStream(path)) {
+            CertificateFactory factory = CertificateFactory.getInstance("X.509");
+            return (X509Certificate) factory.generateCertificate(in);
+        }
+    }
+
+    private static class Message extends TransportMessage<Message> {
+        private Message() {
+        }
+    }
+}

src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutClientAuthenticationTests.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.node.internal.InternalNode;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
+import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ShieldIntegrationTest;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
+import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.junit.Test;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+
+import static org.hamcrest.Matchers.is;
+
+@ClusterScope(numClientNodes = 0, numDataNodes = 1)
+public class PkiWithoutClientAuthenticationTests extends ShieldIntegrationTest {
+
+    private TrustManager[] trustAllCerts = new TrustManager[] {
+            new X509TrustManager() {
+                public X509Certificate[] getAcceptedIssuers() {
+                    return null;
+                }
+
+                public void checkClientTrusted(X509Certificate[] certs, String authType) {
+                }
+
+                public void checkServerTrusted(X509Certificate[] certs, String authType) {
+                }
+            }
+    };
+
+    @Override
+    public boolean sslTransportEnabled() {
+        return true;
+    }
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(InternalNode.HTTP_ENABLED, true)
+                .put(ShieldNettyTransport.TRANSPORT_CLIENT_AUTH_SETTING, false)
+                .put(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, true)
+                .put(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING, false)
+                .put("shield.authc.realms.pki1.type", "pki")
+                .put("shield.authc.realms.pki1.order", "0")
+                .build();
+    }
+
+    @Test
+    public void testThatTransportClientWorks() {
+        Client client = internalCluster().transportClient();
+        assertGreenClusterState(client);
+    }
+
+    @Test
+    public void testThatHttpWorks() throws Exception {
+        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
+        SSLContext sc = SSLContext.getInstance("SSL");
+        sc.init(null, trustAllCerts, new SecureRandom());
+        try (CloseableHttpClient httpClient = HttpClients.custom().setSslcontext(sc).build()) {
+            HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+                    .host("localhost")
+                    .port(((InetSocketTransportAddress)httpServerTransport.boundAddress().publishAddress()).address().getPort())
+                    .protocol("https")
+                    .method("GET")
+                    .path("/_nodes");
+            requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME, new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray())));
+            HttpResponse response = requestBuilder.execute();
+            assertThat(response.getStatusCode(), is(200));
+        }
+    }
+}

src/test/java/org/elasticsearch/shield/authc/pki/PkiWithoutSSLTests.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.pki;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.node.internal.InternalNode;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ShieldIntegrationTest;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
+import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.*;
+
+@ClusterScope(numClientNodes = 0, numDataNodes = 1)
+public class PkiWithoutSSLTests extends ShieldIntegrationTest {
+
+    @Override
+    public boolean sslTransportEnabled() {
+        return false;
+    }
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal) {
+        return ImmutableSettings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(InternalNode.HTTP_ENABLED, true)
+                .put("shield.authc.realms.pki1.type", "pki")
+                .put("shield.authc.realms.pki1.order", "0")
+                .build();
+    }
+
+    @Test
+    public void testThatTransportClientWorks() {
+        Client client = internalCluster().transportClient();
+        assertGreenClusterState(client);
+    }
+
+    @Test
+    public void testThatHttpWorks() throws Exception {
+        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
+        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+            HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+                    .httpTransport(httpServerTransport)
+                    .method("GET")
+                    .path("/_nodes");
+            requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME, new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray())));
+            HttpResponse response = requestBuilder.execute();
+            assertThat(response.getStatusCode(), is(200));
+        }
+    }
+}

src/test/java/org/elasticsearch/shield/authc/support/DnRoleMapperTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.authc.ldap.support;
+package org.elasticsearch.shield.authc.support;
 
 import com.unboundid.ldap.sdk.DN;
 import org.elasticsearch.common.base.Charsets;
@@ -16,7 +16,6 @@ import org.elasticsearch.shield.audit.logfile.CapturingLogger;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.activedirectory.ActiveDirectoryRealm;
 import org.elasticsearch.shield.authc.ldap.LdapRealm;
-import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -40,7 +39,7 @@ import static org.hamcrest.Matchers.*;
 /**
  *
  */
-public class LdapRoleMapperTests extends ElasticsearchTestCase {
+public class DnRoleMapperTests extends ElasticsearchTestCase {
 
     private static final String[] STARK_GROUP_DNS = new String[] {
             //groups can be named by different attributes, depending on the directory,
@@ -78,20 +77,20 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
         Files.write(file, ImmutableList.of("aldlfkjldjdflkjd"), Charsets.UTF_16);
 
         ResourceWatcherService watcherService = new ResourceWatcherService(settings, threadPool);
-        LdapRoleMapper mapper = createMapper(file, watcherService);
+        DnRoleMapper mapper = createMapper(file, watcherService);
         assertThat(mapper.mappingsCount(), is(0));
     }
 
     @Test
     public void testMapper_AutoReload() throws Exception {
-        Path roleMappingFile = Paths.get(LdapRoleMapperTests.class.getResource("role_mapping.yml").toURI());
+        Path roleMappingFile = Paths.get(DnRoleMapperTests.class.getResource("role_mapping.yml").toURI());
         Path file = Files.createTempFile(null, ".yml");
         Files.copy(roleMappingFile, file, StandardCopyOption.REPLACE_EXISTING);
 
         final CountDownLatch latch = new CountDownLatch(1);
 
         ResourceWatcherService watcherService = new ResourceWatcherService(settings, threadPool);
-        LdapRoleMapper mapper = createMapper(file, watcherService);
+        DnRoleMapper mapper = createMapper(file, watcherService);
         mapper.addListener(new RefreshListener() {
             @Override
             public void onRefresh() {
@@ -124,14 +123,14 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
 
     @Test
     public void testMapper_AutoReload_WithParseFailures() throws Exception {
-        Path roleMappingFile = Paths.get(LdapRoleMapperTests.class.getResource("role_mapping.yml").toURI());
+        Path roleMappingFile = Paths.get(DnRoleMapperTests.class.getResource("role_mapping.yml").toURI());
         Path file = Files.createTempFile(null, ".yml");
         Files.copy(roleMappingFile, file, StandardCopyOption.REPLACE_EXISTING);
 
         final CountDownLatch latch = new CountDownLatch(1);
 
         ResourceWatcherService watcherService = new ResourceWatcherService(settings, threadPool);
-        LdapRoleMapper mapper = createMapper(file, watcherService);
+        DnRoleMapper mapper = createMapper(file, watcherService);
         mapper.addListener(new RefreshListener() {
             @Override
             public void onRefresh() {
@@ -158,9 +157,9 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
 
     @Test
     public void testParseFile() throws Exception {
-        Path file = Paths.get(LdapRoleMapperTests.class.getResource("role_mapping.yml").toURI());
+        Path file = Paths.get(DnRoleMapperTests.class.getResource("role_mapping.yml").toURI());
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO);
-        ImmutableMap<DN, Set<String>> mappings = LdapRoleMapper.parseFile(file, logger, "_type", "_name");
+        ImmutableMap<DN, Set<String>> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name");
         assertThat(mappings, notNullValue());
         assertThat(mappings.size(), is(3));
 
@@ -190,7 +189,7 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
     public void testParseFile_Empty() throws Exception {
         Path file = newTempFile().toPath();
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO);
-        ImmutableMap<DN, Set<String>> mappings = LdapRoleMapper.parseFile(file, logger, "_type", "_name");
+        ImmutableMap<DN, Set<String>> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name");
         assertThat(mappings, notNullValue());
         assertThat(mappings.isEmpty(), is(true));
         List<CapturingLogger.Msg> msgs = logger.output(CapturingLogger.Level.WARN);
@@ -202,7 +201,7 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
     public void testParseFile_WhenFileDoesNotExist() throws Exception {
         Path file = new File(randomAsciiOfLength(10)).toPath();
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO);
-        ImmutableMap<DN, Set<String>> mappings = LdapRoleMapper.parseFile(file, logger, "_type", "_name");
+        ImmutableMap<DN, Set<String>> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name");
         assertThat(mappings, notNullValue());
         assertThat(mappings.isEmpty(), is(true));
     }
@@ -214,7 +213,7 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
         Files.write(file, ImmutableList.of("aldlfkjldjdflkjd"), Charsets.UTF_16);
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO);
         try {
-            LdapRoleMapper.parseFile(file, logger, "_type", "_name");
+            DnRoleMapper.parseFile(file, logger, "_type", "_name");
             fail("expected a parse failure");
         } catch (Exception e) {
             this.logger.info("expected", e);
@@ -227,7 +226,7 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
         // writing in utf_16 should cause a parsing error as we try to read the file in utf_8
         Files.write(file, ImmutableList.of("aldlfkjldjdflkjd"), Charsets.UTF_16);
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.INFO);
-        ImmutableMap<DN, Set<String>> mappings = LdapRoleMapper.parseFileLenient(file, logger, "_type", "_name");
+        ImmutableMap<DN, Set<String>> mappings = DnRoleMapper.parseFileLenient(file, logger, "_type", "_name");
         assertThat(mappings, notNullValue());
         assertThat(mappings.isEmpty(), is(true));
         List<CapturingLogger.Msg> msgs = logger.output(CapturingLogger.Level.ERROR);
@@ -239,11 +238,11 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
     public void testYaml() throws IOException {
         File file = this.getResource("role_mapping.yml");
         Settings ldapSettings = ImmutableSettings.settingsBuilder()
-                .put(LdapRoleMapper.ROLE_MAPPING_FILE_SETTING, file.getCanonicalPath())
+                .put(DnRoleMapper.ROLE_MAPPING_FILE_SETTING, file.getCanonicalPath())
                 .build();
         RealmConfig config = new RealmConfig("ldap1", ldapSettings);
 
-        LdapRoleMapper mapper = new LdapRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
+        DnRoleMapper mapper = new DnRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
 
         Set<String> roles = mapper.resolveRoles("", Arrays.asList(STARK_GROUP_DNS));
 
@@ -254,11 +253,11 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
     @Test
     public void testRelativeDN() {
         Settings ldapSettings = ImmutableSettings.builder()
-                .put(LdapRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
+                .put(DnRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, true)
                 .build();
         RealmConfig config = new RealmConfig("ldap1", ldapSettings);
 
-        LdapRoleMapper mapper = new LdapRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
+        DnRoleMapper mapper = new DnRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
 
         Set<String> roles = mapper.resolveRoles("", Arrays.asList(STARK_GROUP_DNS));
         assertThat(roles, hasItems("genius", "billionaire", "playboy", "philanthropist", "shield", "avengers"));
@@ -268,22 +267,22 @@ public class LdapRoleMapperTests extends ElasticsearchTestCase {
     public void testUserDNMapping() throws Exception {
         File file = this.getResource("role_mapping.yml");
         Settings ldapSettings = ImmutableSettings.builder()
-                .put(LdapRoleMapper.ROLE_MAPPING_FILE_SETTING, file.getCanonicalPath())
+                .put(DnRoleMapper.ROLE_MAPPING_FILE_SETTING, file.getCanonicalPath())
-                .put(LdapRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, false)
+                .put(DnRoleMapper.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, false)
                 .build();
         RealmConfig config = new RealmConfig("ldap-userdn-role", ldapSettings);
 
-        LdapRoleMapper mapper = new LdapRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
+        DnRoleMapper mapper = new DnRoleMapper(LdapRealm.TYPE, config, new ResourceWatcherService(settings, threadPool), null);
 
         Set<String> roles = mapper.resolveRoles("cn=Horatio Hornblower,ou=people,o=sevenSeas", Collections.<String>emptyList());
         assertThat(roles, hasItem("avenger"));
     }
 
-    protected LdapRoleMapper createMapper(Path file, ResourceWatcherService watcherService) {
+    protected DnRoleMapper createMapper(Path file, ResourceWatcherService watcherService) {
         Settings realmSettings = ImmutableSettings.builder()
                 .put("files.role_mapping", file.toAbsolutePath())
                 .build();
         RealmConfig config = new RealmConfig("ad-group-mapper-test", realmSettings, settings, env);
-        return new LdapRoleMapper(randomBoolean() ? ActiveDirectoryRealm.TYPE : LdapRealm.TYPE, config, watcherService, null);
+        return new DnRoleMapper(randomBoolean() ? ActiveDirectoryRealm.TYPE : LdapRealm.TYPE, config, watcherService, null);
     }
 }

src/test/java/org/elasticsearch/shield/rest/ShieldRestFilterTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.shield.rest;
 
+import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestFilterChain;
@@ -35,7 +36,7 @@ public class ShieldRestFilterTests extends ElasticsearchTestCase {
         RestController restController = mock(RestController.class);
         channel = mock(RestChannel.class);
         chain = mock(RestFilterChain.class);
-        filter = new ShieldRestFilter(authcService, restController);
+        filter = new ShieldRestFilter(authcService, restController, ImmutableSettings.EMPTY);
         verify(restController).registerFilter(filter);
     }
 

src/test/java/org/elasticsearch/shield/transport/ssl/SslClientAuthTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.node.internal.InternalNode;
 import org.elasticsearch.shield.ssl.ClientSSLService;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.elasticsearch.test.ShieldIntegrationTest;
 import org.elasticsearch.test.ShieldSettingsSource;
 import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
@@ -40,8 +41,8 @@ public class SslClientAuthTests extends ShieldIntegrationTest {
                 .put(super.nodeSettings(nodeOrdinal))
                 // invert the require auth settings
                 .put("shield.transport.ssl", true)
-                .put("shield.http.ssl", true)
+                .put(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, true)
-                .put("shield.http.ssl.client.auth", true)
+                .put(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING, true)
                 .put("transport.profiles.default.shield.ssl.client.auth", false)
                 .put(InternalNode.HTTP_ENABLED, true)
                 .build();

src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.node.internal.InternalNode;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.elasticsearch.test.ShieldIntegrationTest;
 import org.elasticsearch.transport.Transport;
 import org.junit.Test;
@@ -56,7 +57,7 @@ public class SslIntegrationTests extends ShieldIntegrationTest {
     protected Settings nodeSettings(int nodeOrdinal) {
         return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal))
                 .put(InternalNode.HTTP_ENABLED, true)
-                .put("shield.http.ssl", true).build();
+                .put(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, true).build();
     }
 
     @Override

src/test/java/org/elasticsearch/shield/tribe/TribeTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.signature.InternalSignatureService;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.ShieldIntegrationTest;
 import org.elasticsearch.test.ShieldSettingsSource;
@@ -127,7 +128,7 @@ public class TribeTests extends ShieldIntegrationTest {
                         return true;
                     }
                     //forward ssl settings to the tribe clients, same certificates will be used
-                    if (settingKey.startsWith("shield.ssl") || settingKey.equals("shield.transport.ssl") || settingKey.equals("shield.http.ssl")) {
+                    if (settingKey.startsWith("shield.ssl") || settingKey.equals("shield.transport.ssl") || settingKey.equals(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING)) {
                         return true;
                     }
                     //forward the credentials to the tribe clients

src/test/java/org/elasticsearch/test/ShieldSettingsSource.java

@@ -17,6 +17,7 @@ import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.signature.InternalSignatureService;
 import org.elasticsearch.shield.test.ShieldTestUtils;
+import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
 import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
 
@@ -218,7 +219,7 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
 
         ImmutableSettings.Builder builder = settingsBuilder()
                 .put("shield.transport.ssl", sslTransportEnabled)
-                .put("shield.http.ssl", false);
+                .put(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING, false);
 
         if (sslTransportEnabled) {
             builder.put("shield.ssl.keystore.path", store.getPath())

src/test/resources/org/elasticsearch/shield/authc/pki/role_mapping.yml

@@ -0,0 +1,2 @@
+user:
+  - "CN=Elasticsearch Test Node, OU=elasticsearch, O=org"