allow authentication exceptions for unauthorized anonymous users

Today, we always throw an AuthorizationException for an unauthorized user. This is
problematic when anonymous access is enabled and the HTTP client being used
does not support preemptive basic authentication as only the anonymous user
will be used by such a client.

This change adds a setting to allow an AuthenticationException to be thrown for
anonymous users. This will clients such as browsers to work with anonymous
access and authenticated access.

Closes elastic/elasticsearch#853

Original commit: elastic/x-pack-elasticsearch@d338b468c7

src/main/java/org/elasticsearch/shield/ShieldException.java

@@ -7,25 +7,19 @@ package org.elasticsearch.shield;
 
 import org.elasticsearch.ElasticsearchException;
 
-import org.elasticsearch.common.collect.ImmutableMap;
-import org.elasticsearch.common.collect.Lists;
 import org.elasticsearch.common.collect.Tuple;
 
-import java.util.List;
-
 /**
  *
  */
 public class ShieldException extends ElasticsearchException.WithRestHeaders {
 
-    public static final Tuple<String, String[]> BASIC_AUTH_HEADER = Tuple.tuple("WWW-Authenticate", new String[]{"Basic realm=\"" + ShieldPlugin.NAME + "\""});
+    public ShieldException(String msg, Tuple... headers) {
-
+        super(msg, headers);
-    public ShieldException(String msg) {
-        super(msg, BASIC_AUTH_HEADER);
     }
 
-    public ShieldException(String msg, Throwable cause) {
+    public ShieldException(String msg, Throwable cause, Tuple... headers) {
-        super(msg, BASIC_AUTH_HEADER);
+        super(msg, headers);
         initCause(cause);
     }
 }

src/main/java/org/elasticsearch/shield/authc/AnonymousService.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.User;
+
+public class AnonymousService {
+
+    public static final String SETTING_AUTHORIZATION_EXCEPTION_ENABLED = "shield.authc.anonymous.authz_exception";
+    static final String ANONYMOUS_USERNAME = "_es_anonymous_user";
+
+
+    @Nullable
+    private final User anonymousUser;
+    private final boolean authzExceptionEnabled;
+
+    @Inject
+    public AnonymousService(Settings settings) {
+        anonymousUser = resolveAnonymousUser(settings);
+        authzExceptionEnabled = settings.getAsBoolean(SETTING_AUTHORIZATION_EXCEPTION_ENABLED, true);
+    }
+
+    public boolean enabled() {
+        return anonymousUser != null;
+    }
+
+    public boolean isAnonymous(User user) {
+        if (enabled()) {
+            return anonymousUser.equals(user);
+        }
+        return false;
+    }
+
+    public User anonymousUser() {
+        return anonymousUser;
+    }
+
+    public boolean authorizationExceptionsEnabled() {
+        return authzExceptionEnabled;
+    }
+
+    static User resolveAnonymousUser(Settings settings) {
+        String[] roles = settings.getAsArray("shield.authc.anonymous.roles", null);
+        if (roles == null) {
+            return null;
+        }
+        String username = settings.get("shield.authc.anonymous.username", ANONYMOUS_USERNAME);
+        return new User.Simple(username, roles);
+    }
+}

src/main/java/org/elasticsearch/shield/authc/AuthenticationException.java

@@ -5,20 +5,24 @@
  */
 package org.elasticsearch.shield.authc;
 
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.shield.ShieldException;
+import org.elasticsearch.shield.ShieldPlugin;
 
 /**
  *
  */
 public class AuthenticationException extends ShieldException {
 
+    public static final Tuple<String, String[]> BASIC_AUTH_HEADER = Tuple.tuple("WWW-Authenticate", new String[]{"Basic realm=\"" + ShieldPlugin.NAME + "\""});
+
     public AuthenticationException(String msg) {
-        super(msg);
+        super(msg, BASIC_AUTH_HEADER);
     }
 
     public AuthenticationException(String msg, Throwable cause) {
-        super(msg, cause);
+        super(msg, cause, BASIC_AUTH_HEADER);
     }
 
     @Override

src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java

@@ -31,6 +31,7 @@ public class AuthenticationModule extends AbstractShieldModule.Node {
         mapBinder.addBinding(PkiRealm.TYPE).to(PkiRealm.Factory.class).asEagerSingleton();
 
         bind(Realms.class).asEagerSingleton();
+        bind(AnonymousService.class).asEagerSingleton();
         bind(AuthenticationService.class).to(InternalAuthenticationService.class).asEagerSingleton();
     }
 }

src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.shield.authc;
 
 import org.elasticsearch.common.Base64;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -29,7 +28,6 @@ import java.io.IOException;
 public class InternalAuthenticationService extends AbstractComponent implements AuthenticationService {
 
     public static final String SETTING_SIGN_USER_HEADER = "shield.authc.sign_user_header";
-    static final String ANONYMOUS_USERNAME = "_es_anonymous_user";
 
     static final String TOKEN_KEY = "_shield_token";
     static final String USER_KEY = "_shield_user";
@@ -37,30 +35,28 @@ public class InternalAuthenticationService extends AbstractComponent implements
     private final Realms realms;
     private final AuditTrail auditTrail;
     private final CryptoService cryptoService;
+    private final AnonymousService anonymousService;
     private final boolean signUserHeader;
 
-    @Nullable
-    private final User anonymouseUser;
-
     @Inject
-    public InternalAuthenticationService(Settings settings, Realms realms, AuditTrail auditTrail, CryptoService cryptoService) {
+    public InternalAuthenticationService(Settings settings, Realms realms, AuditTrail auditTrail, CryptoService cryptoService, AnonymousService anonymousService) {
         super(settings);
         this.realms = realms;
         this.auditTrail = auditTrail;
         this.cryptoService = cryptoService;
+        this.anonymousService = anonymousService;
         this.signUserHeader = settings.getAsBoolean(SETTING_SIGN_USER_HEADER, true);
-        anonymouseUser = resolveAnonymouseUser(settings);
     }
 
     @Override
     public User authenticate(RestRequest request) throws AuthenticationException {
         AuthenticationToken token = token(request);
         if (token == null) {
-            if (anonymouseUser != null) {
+            if (anonymousService.enabled()) {
                 // we must put the user in the request context, so it'll be copied to the
                 // transport request - without it, the transport will assume system user
-                request.putInContext(USER_KEY, anonymouseUser);
+                request.putInContext(USER_KEY, anonymousService.anonymousUser());
-                return anonymouseUser;
+                return anonymousService.anonymousUser();
             }
             auditTrail.anonymousAccessDenied(request);
             throw new AuthenticationException("missing authentication token for REST request [" + request.uri() + "]");
@@ -138,15 +134,6 @@ public class InternalAuthenticationService extends AbstractComponent implements
         }
     }
 
-    static User resolveAnonymouseUser(Settings settings) {
-        String[] roles = settings.getAsArray("shield.authc.anonymous.roles", null);
-        if (roles == null) {
-            return null;
-        }
-        String username = settings.get("shield.authc.anonymous.username", ANONYMOUS_USERNAME);
-        return new User.Simple(username, roles);
-    }
-
     /**
      * Authenticates the user associated with the given request by delegating the authentication to
      * the configured realms. Each realm that supports the given token will be asked to perform authentication,
@@ -173,8 +160,8 @@ public class InternalAuthenticationService extends AbstractComponent implements
             if (fallbackUser != null) {
                 return fallbackUser;
             }
-            if (anonymouseUser != null) {
+            if (anonymousService.enabled()) {
-                return anonymouseUser;
+                return anonymousService.anonymousUser();
             }
             auditTrail.anonymousAccessDenied(action, message);
             throw new AuthenticationException("missing authentication token for action [" + action + "]");

src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java

@@ -23,6 +23,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.action.SearchServiceTransportAction;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.audit.AuditTrail;
+import org.elasticsearch.shield.authc.AnonymousService;
+import org.elasticsearch.shield.authc.AuthenticationException;
 import org.elasticsearch.shield.authz.indicesresolver.DefaultIndicesResolver;
 import org.elasticsearch.shield.authz.indicesresolver.IndicesResolver;
 import org.elasticsearch.shield.authz.store.RolesStore;
@@ -40,9 +42,10 @@ public class InternalAuthorizationService extends AbstractComponent implements A
     private final RolesStore rolesStore;
     private final AuditTrail auditTrail;
     private final IndicesResolver[] indicesResolvers;
+    private final AnonymousService anonymousService;
 
     @Inject
-    public InternalAuthorizationService(Settings settings, RolesStore rolesStore, ClusterService clusterService, AuditTrail auditTrail) {
+    public InternalAuthorizationService(Settings settings, RolesStore rolesStore, ClusterService clusterService, AuditTrail auditTrail, AnonymousService anonymousService) {
         super(settings);
         this.rolesStore = rolesStore;
         this.clusterService = clusterService;
@@ -50,6 +53,7 @@ public class InternalAuthorizationService extends AbstractComponent implements A
         this.indicesResolvers = new IndicesResolver[] {
                 new DefaultIndicesResolver(this)
         };
+        this.anonymousService = anonymousService;
     }
 
     @Override
@@ -231,6 +235,12 @@ public class InternalAuthorizationService extends AbstractComponent implements A
 
     private AuthorizationException denial(User user, String action, TransportRequest request) {
         auditTrail.accessDenied(user, action, request);
+        // Special case for anonymous user
+        if (anonymousService.isAnonymous(user)) {
+            if (!anonymousService.authorizationExceptionsEnabled()) {
+                throw new AuthenticationException("action [" + action + "] requires authentication");
+            }
+        }
         return new AuthorizationException("action [" + action + "] is unauthorized for user [" + user.principal() + "]");
     }
 

src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc;
+
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.hamcrest.Matchers.*;
+
+public class AnonymousUserHolderTests extends ElasticsearchTestCase {
+
+    @Test
+    public void testResolveAnonymousUser() throws Exception {
+        Settings settings = Settings.builder()
+                .put("shield.authc.anonymous.username", "anonym1")
+                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
+                .build();
+        User user = AnonymousService.resolveAnonymousUser(settings);
+        assertThat(user, notNullValue());
+        assertThat(user.principal(), equalTo("anonym1"));
+        assertThat(user.roles(), arrayContainingInAnyOrder("r1", "r2", "r3"));
+
+        settings = Settings.builder()
+                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
+                .build();
+        user = AnonymousService.resolveAnonymousUser(settings);
+        assertThat(user, notNullValue());
+        assertThat(user.principal(), equalTo(AnonymousService.ANONYMOUS_USERNAME));
+        assertThat(user.roles(), arrayContainingInAnyOrder("r1", "r2", "r3"));
+    }
+
+    @Test
+    public void testResolveAnonymousUser_NoSettings() throws Exception {
+        Settings settings = randomBoolean() ?
+                Settings.EMPTY :
+                Settings.builder().put("shield.authc.anonymous.username", "user1").build();
+        User user = AnonymousService.resolveAnonymousUser(settings);
+        assertThat(user, nullValue());
+    }
+
+    @Test
+    public void testWhenAnonymousDisabled() {
+        AnonymousService anonymousService = new AnonymousService(Settings.EMPTY);
+        assertThat(anonymousService.enabled(), is(false));
+        assertThat(anonymousService.isAnonymous(new User.Simple(randomAsciiOfLength(10), randomAsciiOfLength(5))), is(false));
+        assertThat(anonymousService.anonymousUser(), nullValue());
+        assertThat(anonymousService.authorizationExceptionsEnabled(), is(true));
+    }
+
+    @Test
+    public void testWhenAnonymousEnabled() throws Exception {
+        Settings settings = Settings.builder()
+                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
+                .build();
+        AnonymousService anonymousService = new AnonymousService(settings);
+        assertThat(anonymousService.enabled(), is(true));
+        assertThat(anonymousService.anonymousUser(), notNullValue());
+        assertThat(anonymousService.isAnonymous(anonymousService.anonymousUser()), is(true));
+        assertThat(anonymousService.authorizationExceptionsEnabled(), is(true));
+
+        // make sure check works with serialization
+        BytesStreamOutput output = new BytesStreamOutput();
+        User.writeTo(anonymousService.anonymousUser(), output);
+        User anonymousSerialized = User.readFrom(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytes())));
+        assertThat(anonymousService.isAnonymous(anonymousSerialized), is(true));
+    }
+
+    @Test
+    public void testDisablingAuthorizationExceptions() {
+        Settings settings = Settings.builder()
+                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
+                .put(AnonymousService.SETTING_AUTHORIZATION_EXCEPTION_ENABLED, false)
+                .build();
+        AnonymousService holder = new AnonymousService(settings);
+        assertThat(holder.authorizationExceptionsEnabled(), is(false));
+    }
+}

src/test/java/org/elasticsearch/shield/authc/AnonymousUserTests.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc;
+
+import com.google.common.base.Charsets;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ShieldIntegrationTest;
+import org.junit.Test;
+
+import java.io.InputStreamReader;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.*;
+
+public class AnonymousUserTests extends ShieldIntegrationTest {
+
+    private boolean authorizationExceptionsEnabled = randomBoolean();
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(Node.HTTP_ENABLED, true)
+                .put("shield.authc.anonymous.roles", "anonymous")
+                .put(AnonymousService.SETTING_AUTHORIZATION_EXCEPTION_ENABLED, authorizationExceptionsEnabled)
+                .build();
+    }
+
+    @Override
+    public boolean sslTransportEnabled() {
+        return false;
+    }
+
+    @Override
+    public String configRoles() {
+        return super.configRoles() + "\n" +
+                "anonymous:\n" +
+                "  indices:\n" +
+                "    '*': READ";
+    }
+
+    @Test
+    public void testAnonymousViaHttp() throws Exception {
+        try (CloseableHttpClient client = HttpClients.createDefault();
+             CloseableHttpResponse response = client.execute(new HttpGet(getNodeUrl() + "_nodes"))) {
+            int statusCode = response.getStatusLine().getStatusCode();
+            String data = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), Charsets.UTF_8));
+            if (authorizationExceptionsEnabled) {
+                assertThat(statusCode, is(403));
+                assertThat(response.getFirstHeader("WWW-Authenticate"), nullValue());
+                assertThat(data, containsString("authorization_exception"));
+            } else {
+                assertThat(statusCode, is(401));
+                assertThat(response.getFirstHeader("WWW-Authenticate"), notNullValue());
+                assertThat(response.getFirstHeader("WWW-Authenticate").getValue(), containsString("Basic"));
+                assertThat(data, containsString("authentication_exception"));
+            }
+        }
+    }
+
+    private String getNodeUrl() {
+        TransportAddress transportAddress = internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddress();
+        assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
+        InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
+        return String.format(Locale.ROOT, "http://%s:%s/", "localhost", inetSocketTransportAddress.address().getPort());
+    }
+}

src/test/java/org/elasticsearch/shield/authc/InternalAuthenticationServiceTests.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.shield.authc;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -49,6 +49,7 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
     AuditTrail auditTrail;
     AuthenticationToken token;
     CryptoService cryptoService;
+    AnonymousService anonymousService;
 
     @Before
     public void init() throws Exception {
@@ -70,7 +71,8 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
         cryptoService = mock(CryptoService.class);
 
         auditTrail = mock(AuditTrail.class);
-        service = new InternalAuthenticationService(Settings.EMPTY, realms, auditTrail, cryptoService);
+        anonymousService = mock(AnonymousService.class);
+        service = new InternalAuthenticationService(Settings.EMPTY, realms, auditTrail, cryptoService, anonymousService);
     }
 
     @Test @SuppressWarnings("unchecked")
@@ -335,7 +337,7 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
     @Test
     public void testAutheticate_Transport_ContextAndHeader_NoSigning() throws Exception {
         Settings settings = Settings.builder().put(InternalAuthenticationService.SETTING_SIGN_USER_HEADER, false).build();
-        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService);
+        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService, anonymousService);
 
         User user1 = new User.Simple("username", "r1", "r2");
         when(firstRealm.supports(token)).thenReturn(true);
@@ -400,44 +402,17 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
         assertThat(message.getHeader(InternalAuthenticationService.USER_KEY), equalTo((Object) "_signed_user"));
     }
 
-    @Test
-    public void testResolveAnonymousUser() throws Exception {
-        Settings settings = Settings.builder()
-                .put("shield.authc.anonymous.username", "anonym1")
-                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
-                .build();
-        User user = InternalAuthenticationService.resolveAnonymouseUser(settings);
-        assertThat(user, notNullValue());
-        assertThat(user.principal(), equalTo("anonym1"));
-        assertThat(user.roles(), arrayContainingInAnyOrder("r1", "r2", "r3"));
-
-        settings = Settings.builder()
-                .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
-                .build();
-        user = InternalAuthenticationService.resolveAnonymouseUser(settings);
-        assertThat(user, notNullValue());
-        assertThat(user.principal(), equalTo(InternalAuthenticationService.ANONYMOUS_USERNAME));
-        assertThat(user.roles(), arrayContainingInAnyOrder("r1", "r2", "r3"));
-    }
-
-    @Test
-    public void testResolveAnonymousUser_NoSettings() throws Exception {
-        Settings settings = randomBoolean() ?
-                Settings.EMPTY :
-                Settings.builder().put("shield.authc.anonymous.username", "user1").build();
-        User user = InternalAuthenticationService.resolveAnonymouseUser(settings);
-        assertThat(user, nullValue());
-    }
-
     @Test
     public void testAnonymousUser_Rest() throws Exception {
-        String username = randomBoolean() ? InternalAuthenticationService.ANONYMOUS_USERNAME : "user1";
+        String username = randomBoolean() ? AnonymousService.ANONYMOUS_USERNAME : "user1";
         Settings.Builder builder = Settings.builder()
                 .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3");
-        if (username != InternalAuthenticationService.ANONYMOUS_USERNAME) {
+        if (username != AnonymousService.ANONYMOUS_USERNAME) {
             builder.put("shield.authc.anonymous.username", username);
         }
-        service = new InternalAuthenticationService(builder.build(), realms, auditTrail, cryptoService);
+        Settings settings = builder.build();
+        AnonymousService holder = new AnonymousService(settings);
+        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService, holder);
 
         RestRequest request = new FakeRestRequest();
 
@@ -454,13 +429,13 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
         Settings settings = Settings.builder()
                 .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
                 .build();
-        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService);
+        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService, new AnonymousService(settings));
 
         InternalMessage message = new InternalMessage();
 
         User user = service.authenticate("_action", message, null);
         assertThat(user, notNullValue());
-        assertThat(user.principal(), equalTo(InternalAuthenticationService.ANONYMOUS_USERNAME));
+        assertThat(user.principal(), equalTo(AnonymousService.ANONYMOUS_USERNAME));
         assertThat(user.roles(), arrayContainingInAnyOrder("r1", "r2", "r3"));
     }
 
@@ -469,7 +444,7 @@ public class InternalAuthenticationServiceTests extends ElasticsearchTestCase {
         Settings settings = Settings.builder()
                 .putArray("shield.authc.anonymous.roles", "r1", "r2", "r3")
                 .build();
-        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService);
+        service = new InternalAuthenticationService(settings, realms, auditTrail, cryptoService, new AnonymousService(settings));
 
         InternalMessage message = new InternalMessage();
 

src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordTokenTests.java

@@ -72,7 +72,7 @@ public class UsernamePasswordTokenTests extends ElasticsearchTestCase {
     }
 
     @Test
-    public void testThatAuthorizationExceptionContainsResponseHeaders() {
+    public void testThatAuthenticationExceptionContainsResponseHeaders() {
         TransportRequest request = new TransportRequest() {};
         String header = "BasicBroken";
         request.putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, header);

src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java

@@ -0,0 +1,335 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.search.*;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.ImmutableList;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.action.SearchServiceTransportAction;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.audit.AuditTrail;
+import org.elasticsearch.shield.authc.AnonymousService;
+import org.elasticsearch.shield.authc.AuthenticationException;
+import org.elasticsearch.shield.authz.store.RolesStore;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.elasticsearch.transport.TransportRequest;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class InternalAuthorizationServiceTests extends ElasticsearchTestCase {
+
+    private AuditTrail auditTrail;
+    private RolesStore rolesStore;
+    private ClusterService clusterService;
+    private InternalAuthorizationService internalAuthorizationService;
+
+    @Before
+    public void setup() {
+        rolesStore = mock(RolesStore.class);
+        clusterService = mock(ClusterService.class);
+        auditTrail = mock(AuditTrail.class);
+        AnonymousService anonymousService = new AnonymousService(Settings.EMPTY);
+        internalAuthorizationService = new InternalAuthorizationService(Settings.EMPTY, rolesStore, clusterService, auditTrail, anonymousService);
+    }
+
+    @Test
+    public void testActionsSystemUserIsAuthorized() {
+        TransportRequest request = mock(TransportRequest.class);
+
+        // A failure would throw an exception
+        internalAuthorizationService.authorize(User.SYSTEM, "indices:monitor/whatever", request);
+        verify(auditTrail).accessGranted(User.SYSTEM, "indices:monitor/whatever", request);
+
+        internalAuthorizationService.authorize(User.SYSTEM, "internal:whatever", request);
+        verify(auditTrail).accessGranted(User.SYSTEM, "internal:whatever", request);
+    }
+
+    @Test
+    public void testIndicesActionsAreNotAuthorized() {
+        TransportRequest request = mock(TransportRequest.class);
+        try {
+            internalAuthorizationService.authorize(User.SYSTEM, "indices:", request);
+            fail("action beginning with indices should have failed");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:] is unauthorized for user [" + User.SYSTEM.principal() + "]"));
+            verify(auditTrail).accessDenied(User.SYSTEM, "indices:", request);
+        }
+    }
+
+    @Test
+    public void testClusterAdminActionsAreNotAuthorized() {
+        TransportRequest request = mock(TransportRequest.class);
+        try {
+            internalAuthorizationService.authorize(User.SYSTEM, "cluster:admin/whatever", request);
+            fail("action beginning with cluster:admin/whatever should have failed");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [cluster:admin/whatever] is unauthorized for user [" + User.SYSTEM.principal() + "]"));
+            verify(auditTrail).accessDenied(User.SYSTEM, "cluster:admin/whatever", request);
+        }
+    }
+
+    @Test
+    public void testClusterAdminSnapshotStatusActionIsNotAuthorized() {
+        TransportRequest request = mock(TransportRequest.class);
+        try {
+            internalAuthorizationService.authorize(User.SYSTEM, "cluster:admin/snapshot/status", request);
+            fail("action beginning with cluster:admin/snapshot/status should have failed");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [cluster:admin/snapshot/status] is unauthorized for user [" + User.SYSTEM.principal() + "]"));
+            verify(auditTrail).accessDenied(User.SYSTEM, "cluster:admin/snapshot/status", request);
+        }
+    }
+
+    @Test
+    public void testNoRolesCausesDenial() {
+        TransportRequest request = mock(TransportRequest.class);
+        User user = new User.Simple("test user");
+        try {
+            internalAuthorizationService.authorize(user, "indices:a", request);
+            fail("user without roles should be denied");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, "indices:a", request);
+        }
+    }
+
+    @Test
+    public void testUnknownRoleCausesDenial() {
+        TransportRequest request = mock(TransportRequest.class);
+        User user = new User.Simple("test user", "non-existent-role");
+        try {
+            internalAuthorizationService.authorize(user, "indices:a", request);
+            fail("user with unknown role only should have been denied");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, "indices:a", request);
+        }
+    }
+
+    @Test
+    public void testThatNonIndicesAndNonClusterActionIsDenied() {
+        TransportRequest request = mock(TransportRequest.class);
+        User user = new User.Simple("test user", "a_all");
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_role").add(Privilege.Index.ALL, "a").build());
+
+        try {
+            internalAuthorizationService.authorize(user, "whatever", request);
+            fail("non indices and non cluster requests should be denied");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [whatever] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, "whatever", request);
+        }
+    }
+
+    @Test
+    public void testThatRoleWithNoIndicesIsDenied() {
+        TransportRequest request = new IndicesExistsRequest("a");
+        User user = new User.Simple("test user", "no_indices");
+        when(rolesStore.role("no_indices")).thenReturn(Permission.Global.Role.builder("no_indices").set(Privilege.Cluster.action("")).build());
+
+        try {
+            internalAuthorizationService.authorize(user, "indices:a", request);
+            fail("user only has cluster roles so indices requests should fail");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, "indices:a", request);
+        }
+    }
+
+    @Test
+    public void testScrollRelatedRequestsAllowed() {
+        User user = new User.Simple("test user", "a_all");
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_role").add(Privilege.Index.ALL, "a").build());
+
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        internalAuthorizationService.authorize(user, ClearScrollAction.NAME, clearScrollRequest);
+        verify(auditTrail).accessGranted(user, ClearScrollAction.NAME, clearScrollRequest);
+
+        SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
+        internalAuthorizationService.authorize(user, SearchScrollAction.NAME, searchScrollRequest);
+        verify(auditTrail).accessGranted(user, SearchScrollAction.NAME, searchScrollRequest);
+
+        // We have to use a mock request for other Scroll actions as the actual requests are package private to SearchServiceTransportAction
+        TransportRequest request = mock(TransportRequest.class);
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request);
+
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.SCAN_SCROLL_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.SCAN_SCROLL_ACTION_NAME, request);
+
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.FETCH_ID_SCROLL_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.FETCH_ID_SCROLL_ACTION_NAME, request);
+
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.QUERY_FETCH_SCROLL_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.QUERY_FETCH_SCROLL_ACTION_NAME, request);
+
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.QUERY_SCROLL_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.QUERY_SCROLL_ACTION_NAME, request);
+
+        internalAuthorizationService.authorize(user, SearchServiceTransportAction.FREE_CONTEXT_SCROLL_ACTION_NAME, request);
+        verify(auditTrail).accessGranted(user, SearchServiceTransportAction.FREE_CONTEXT_SCROLL_ACTION_NAME, request);
+    }
+
+    @Test
+    public void testAuthorizeIndicesFailures() {
+        TransportRequest request = new IndicesExistsRequest("b");
+        ClusterState state = mock(ClusterState.class);
+        User user = new User.Simple("test user", "a_all");
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_all").add(Privilege.Index.ALL, "a").build());
+        when(clusterService.state()).thenReturn(state);
+        when(state.metaData()).thenReturn(MetaData.EMPTY_META_DATA);
+
+        try {
+            internalAuthorizationService.authorize(user, "indices:a", request);
+            fail("indices request for b should be denied since there is no such index");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, "indices:a", request);
+            verify(clusterService, times(2)).state();
+            verify(state, times(2)).metaData();
+        }
+    }
+
+    @Test
+    public void testCreateIndexWithAliasWithoutPermissions() {
+        CreateIndexRequest request = new CreateIndexRequest("a");
+        request.alias(new Alias("a2"));
+        ClusterState state = mock(ClusterState.class);
+        User user = new User.Simple("test user", "a_all");
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_all").add(Privilege.Index.ALL, "a").build());
+        when(clusterService.state()).thenReturn(state);
+        when(state.metaData()).thenReturn(MetaData.EMPTY_META_DATA);
+
+        try {
+            internalAuthorizationService.authorize(user, CreateIndexAction.NAME, request);
+            fail("indices creation request with alias should be denied since user does not have permission to alias");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [" + IndicesAliasesAction.NAME + "] is unauthorized for user [test user]"));
+            verify(auditTrail).accessDenied(user, IndicesAliasesAction.NAME, request);
+            verify(clusterService).state();
+            verify(state).metaData();
+        }
+    }
+
+    @Test
+    public void testCreateIndexWithAlias() {
+        CreateIndexRequest request = new CreateIndexRequest("a");
+        request.alias(new Alias("a2"));
+        ClusterState state = mock(ClusterState.class);
+        User user = new User.Simple("test user", "a_all");
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_all").add(Privilege.Index.ALL, "a", "a2").build());
+        when(clusterService.state()).thenReturn(state);
+        when(state.metaData()).thenReturn(MetaData.EMPTY_META_DATA);
+
+        internalAuthorizationService.authorize(user, CreateIndexAction.NAME, request);
+
+        verify(auditTrail).accessGranted(user, CreateIndexAction.NAME, request);
+        verifyNoMoreInteractions(auditTrail);
+        verify(clusterService).state();
+        verify(state).metaData();
+    }
+
+    @Test
+    public void testIndicesAliasesWithNoRolesUser() {
+        User user = new User.Simple("test user");
+
+        ImmutableList<String> list = internalAuthorizationService.authorizedIndicesAndAliases(user, "");
+        assertThat(list.isEmpty(), is(true));
+    }
+
+    @Test
+    public void testIndicesAliasesWithUserHavingRoles() {
+        User user = new User.Simple("test user", "a_star", "b");
+        ClusterState state = mock(ClusterState.class);
+        when(rolesStore.role("a_star")).thenReturn(Permission.Global.Role.builder("a_star").add(Privilege.Index.ALL, "a*").build());
+        when(rolesStore.role("b")).thenReturn(Permission.Global.Role.builder("a_star").add(Privilege.Index.SEARCH, "b").build());
+        when(clusterService.state()).thenReturn(state);
+        Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
+        when(state.metaData()).thenReturn(MetaData.builder()
+                .put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
+                .put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
+                .put(new IndexMetaData.Builder("aaaaaa").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
+                .put(new IndexMetaData.Builder("bbbbb").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
+                .put(new IndexMetaData.Builder("b")
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder("ab").build())
+                        .putAlias(new AliasMetaData.Builder("ba").build())
+                        .build(), true)
+                .build());
+
+        ImmutableList<String> list = internalAuthorizationService.authorizedIndicesAndAliases(user, SearchAction.NAME);
+        assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
+        assertThat(list, not(contains("bbbbb")));
+        assertThat(list, not(contains("ba")));
+    }
+
+    @Test
+    public void testDenialForAnonymousUser() {
+        TransportRequest request = new IndicesExistsRequest("b");
+        ClusterState state = mock(ClusterState.class);
+        AnonymousService anonymousService = new AnonymousService(Settings.builder().put("shield.authc.anonymous.roles", "a_all").build());
+        internalAuthorizationService = new InternalAuthorizationService(Settings.EMPTY, rolesStore, clusterService, auditTrail, anonymousService);
+
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_all").add(Privilege.Index.ALL, "a").build());
+        when(clusterService.state()).thenReturn(state);
+        when(state.metaData()).thenReturn(MetaData.EMPTY_META_DATA);
+
+        try {
+            internalAuthorizationService.authorize(anonymousService.anonymousUser(), "indices:a", request);
+            fail("indices request for b should be denied since there is no such index");
+        } catch (AuthorizationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] is unauthorized for user [" + anonymousService.anonymousUser().principal() + "]"));
+            verify(auditTrail).accessDenied(anonymousService.anonymousUser(), "indices:a", request);
+            verify(clusterService, times(2)).state();
+            verify(state, times(2)).metaData();
+        }
+    }
+
+    @Test
+    public void testDenialForAnonymousUserAuthorizationExceptionDisabled() {
+        TransportRequest request = new IndicesExistsRequest("b");
+        ClusterState state = mock(ClusterState.class);
+        AnonymousService anonymousService = new AnonymousService(Settings.builder()
+                .put("shield.authc.anonymous.roles", "a_all")
+                .put(AnonymousService.SETTING_AUTHORIZATION_EXCEPTION_ENABLED, false)
+                .build());
+        internalAuthorizationService = new InternalAuthorizationService(Settings.EMPTY, rolesStore, clusterService, auditTrail, anonymousService);
+
+        when(rolesStore.role("a_all")).thenReturn(Permission.Global.Role.builder("a_all").add(Privilege.Index.ALL, "a").build());
+        when(clusterService.state()).thenReturn(state);
+        when(state.metaData()).thenReturn(MetaData.EMPTY_META_DATA);
+
+        try {
+            internalAuthorizationService.authorize(anonymousService.anonymousUser(), "indices:a", request);
+            fail("indices request for b should be denied since there is no such index");
+        } catch (AuthenticationException e) {
+            assertThat(e.getMessage(), containsString("action [indices:a] requires authentication"));
+            verify(auditTrail).accessDenied(anonymousService.anonymousUser(), "indices:a", request);
+            verify(clusterService, times(2)).state();
+            verify(state, times(2)).metaData();
+        }
+    }
+}

src/test/java/org/elasticsearch/shield/test/ShieldAssertions.java

@@ -6,17 +6,17 @@
 package org.elasticsearch.shield.test;
 
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.shield.ShieldException;
+import org.elasticsearch.shield.authc.AuthenticationException;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
 public class ShieldAssertions {
 
-    public static void assertContainsWWWAuthenticateHeader(ShieldException e) {
+    public static void assertContainsWWWAuthenticateHeader(AuthenticationException e) {
         assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
         assertThat(e.getHeaders(), hasKey("WWW-Authenticate"));
         assertThat(e.getHeaders().get("WWW-Authenticate"), hasSize(1));
-        assertThat(e.getHeaders().get("WWW-Authenticate").get(0), is(ShieldException.BASIC_AUTH_HEADER.v2()[0]));
+        assertThat(e.getHeaders().get("WWW-Authenticate").get(0), is(AuthenticationException.BASIC_AUTH_HEADER.v2()[0]));
     }
 }