Merge branch 'master' into kibana-shield-skip-ssl-check

Original commit: elastic/x-pack-elasticsearch@0862f14c8f

elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/LicenseVerificationTests.java

@@ -43,7 +43,7 @@ public class LicenseVerificationTests extends ESTestCase {
 
         final License tamperedLicense = License.builder()
                 .fromLicenseSpec(license, license.signature())
-                .expiryDate(license.expiryDate() + 10 * 24 * 60 * 60 * 1000l)
+                .expiryDate(license.expiryDate() + 10 * 24 * 60 * 60 * 1000L)
                 .validate()
                 .build();
 

elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/KeyPairGenerationToolTests.java

@@ -5,17 +5,17 @@
  */
 package org.elasticsearch.license.licensor.tools;
 
-import org.apache.commons.cli.MissingOptionException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
 import org.elasticsearch.common.cli.CliTool.Command;
 import org.elasticsearch.common.cli.CliTool.ExitStatus;
 import org.elasticsearch.common.cli.CliToolTestCase;
+import org.elasticsearch.common.cli.UserError;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.licensor.tools.KeyPairGeneratorTool.KeyGenerator;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.core.IsEqual.equalTo;
@@ -24,21 +24,17 @@ public class KeyPairGenerationToolTests extends CliToolTestCase {
     public void testParsingMissingPath() throws Exception {
         KeyPairGeneratorTool keyPairGeneratorTool = new KeyPairGeneratorTool();
         Path tempFile = createTempFile();
-        try {
+        UserError e = expectThrows(UserError.class, () -> {
             keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME,
                 new String[]{"--privateKeyPath", tempFile.toAbsolutePath().toString()});
-            fail("no public key path provided");
+        });
-        } catch (MissingOptionException e) {
         assertThat(e.getMessage(), containsString("pub"));
-        }
+        e = expectThrows(UserError.class, () -> {
-        try {
             keyPairGeneratorTool.parse(KeyPairGeneratorTool.NAME,
                     new String[] { "--publicKeyPath", tempFile.toAbsolutePath().toString() });
-            fail("no private key path provided");
+        });
-        } catch (MissingOptionException e) {
         assertThat(e.getMessage(), containsString("pri"));
     }
-    }
 
     public void testParsingNeverOverrideKey() throws Exception {
         KeyPairGeneratorTool keyPairGeneratorTool = new KeyPairGeneratorTool();

elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseGenerationToolTests.java

@@ -5,10 +5,14 @@
  */
 package org.elasticsearch.license.licensor.tools;
 
-import org.apache.commons.cli.MissingOptionException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
 import org.elasticsearch.common.cli.CliTool.Command;
 import org.elasticsearch.common.cli.CliTool.ExitStatus;
 import org.elasticsearch.common.cli.CliToolTestCase;
+import org.elasticsearch.common.cli.UserError;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.core.License;
@@ -16,10 +20,6 @@ import org.elasticsearch.license.licensor.TestUtils;
 import org.elasticsearch.license.licensor.tools.LicenseGeneratorTool.LicenseGenerator;
 import org.junit.Before;
 
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.core.IsEqual.equalTo;
@@ -72,16 +72,14 @@ public class LicenseGenerationToolTests extends CliToolTestCase {
         TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT);
         LicenseGeneratorTool licenseGeneratorTool = new LicenseGeneratorTool();
         boolean pubKeyMissing = randomBoolean();
-        try {
+        UserError e = expectThrows(UserError.class, () -> {
             licenseGeneratorTool.parse(LicenseGeneratorTool.NAME,
                 new String[]{"--license", TestUtils.generateLicenseSpecString(inputLicenseSpec),
                     ((pubKeyMissing) ? "--privateKeyPath" : "--publicKeyPath"),
                     ((pubKeyMissing) ? priKeyPath.toString() : pubKeyPath.toString())});
-            fail("missing argument: " + ((pubKeyMissing) ? "publicKeyPath" : "privateKeyPath") + " should throw an exception");
+        });
-        } catch (MissingOptionException e) {
         assertThat(e.getMessage(), containsString((pubKeyMissing) ? "pub" : "pri"));
     }
-    }
 
     public void testParsingSimple() throws Exception {
         TestUtils.LicenseSpec inputLicenseSpec = TestUtils.generateRandomLicenseSpec(License.VERSION_CURRENT);

elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/tools/LicenseVerificationToolTests.java

@@ -5,10 +5,16 @@
  */
 package org.elasticsearch.license.licensor.tools;
 
-import org.apache.commons.cli.MissingOptionException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.elasticsearch.common.cli.CliTool.Command;
 import org.elasticsearch.common.cli.CliTool.ExitStatus;
 import org.elasticsearch.common.cli.CliToolTestCase;
+import org.elasticsearch.common.cli.UserError;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.Environment;
@@ -18,12 +24,6 @@ import org.elasticsearch.license.licensor.tools.LicenseVerificationTool.LicenseV
 import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.core.IsEqual.equalTo;
@@ -52,13 +52,12 @@ public class LicenseVerificationToolTests extends CliToolTestCase {
     public void testParsingMissingPublicKeyPath() throws Exception {
         License inputLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(1), pubKeyPath, priKeyPath);
         LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool();
-        try {
+        UserError e = expectThrows(UserError.class, () -> {
             licenseVerificationTool.parse(LicenseVerificationTool.NAME,
                     new String[] { "--license", TestUtils.dumpLicense(inputLicense) });
-        } catch (MissingOptionException e) {
+        });
         assertThat(e.getMessage(), containsString("pub"));
     }
-    }
 
     public void testParsingNonExistentPublicKeyPath() throws Exception {
         LicenseVerificationTool licenseVerificationTool = new LicenseVerificationTool();

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchTests.java

@@ -96,8 +96,8 @@ public class ScriptConditionSearchTests extends AbstractWatcherIntegrationTestCa
         hit.score(1f);
         hit.shard(new SearchShardTarget("a", new Index("a", "testUUID"), 0));
 
-        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(new InternalSearchHits(new InternalSearchHit[]{hit}, 1l, 1f), null, null, null, false, false);
+        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(new InternalSearchHits(new InternalSearchHit[]{hit}, 1L, 1f), null, null, null, false, false);
-        SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500L, new ShardSearchFailure[0]);
 
         WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response));
         assertThat(condition.execute(ctx).met(), is(true));

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionTests.java

@@ -61,7 +61,7 @@ public class ScriptConditionTests extends ESTestCase {
     public void testExecute() throws Exception {
         ScriptServiceProxy scriptService = getScriptServiceProxy(tp);
         ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.payload.hits.total > 1").build()), logger, scriptService);
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
         assertFalse(condition.execute(ctx).met());
     }
@@ -70,7 +70,7 @@ public class ScriptConditionTests extends ESTestCase {
         ScriptServiceProxy scriptService = getScriptServiceProxy(tp);
         Script script = Script.inline("ctx.payload.hits.total > threshold").lang(Script.DEFAULT_LANG).params(singletonMap("threshold", 1)).build();
         ExecutableScriptCondition executable = new ExecutableScriptCondition(new ScriptCondition(script), logger, scriptService);
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
         assertFalse(executable.execute(ctx).met());
     }
@@ -85,7 +85,7 @@ public class ScriptConditionTests extends ESTestCase {
         ScriptCondition condition = factory.parseCondition("_watch", parser);
         ExecutableScriptCondition executable = factory.createExecutable(condition);
 
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
 
         assertFalse(executable.execute(ctx).met());
@@ -163,7 +163,7 @@ public class ScriptConditionTests extends ESTestCase {
     public void testScriptConditionThrowException() throws Exception {
         ScriptServiceProxy scriptService = getScriptServiceProxy(tp);
         ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("assert false").build()), logger, scriptService);
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
         ScriptCondition.Result result = condition.execute(ctx);
         assertThat(result, notNullValue());
@@ -175,7 +175,7 @@ public class ScriptConditionTests extends ESTestCase {
     public void testScriptConditionReturnObject() throws Exception {
         ScriptServiceProxy scriptService = getScriptServiceProxy(tp);
         ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("return new Object()").build()), logger, scriptService);
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
         ScriptCondition.Result result = condition.execute(ctx);
         assertThat(result, notNullValue());
@@ -187,7 +187,7 @@ public class ScriptConditionTests extends ESTestCase {
     public void testScriptConditionAccessCtx() throws Exception {
         ScriptServiceProxy scriptService = getScriptServiceProxy(tp);
         ExecutableScriptCondition condition = new ExecutableScriptCondition(new ScriptCondition(Script.inline("ctx.trigger.scheduled_time.getMillis() < new Date().time ").build()), logger, scriptService);
-        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500l, new ShardSearchFailure[0]);
+        SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]);
         WatchExecutionContext ctx = mockExecutionContext("_name", new DateTime(DateTimeZone.UTC), new Payload.XContent(response));
         Thread.sleep(10);
         assertThat(condition.execute(ctx).met(), is(true));

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/TransformIntegrationTests.java

@@ -119,13 +119,13 @@ public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCas
 
         SearchResponse response = client().prepareSearch("output1").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1));
         assertThat(response.getHits().getAt(0).sourceAsMap().get("key3").toString(), equalTo("20"));
 
         response = client().prepareSearch("output2").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1));
         assertThat(response.getHits().getAt(0).sourceAsMap().get("key3").toString(), equalTo("20"));
     }
@@ -168,12 +168,12 @@ public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCas
 
         SearchResponse response = client().prepareSearch("output1").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsString(), containsString("mytestresult"));
 
         response = client().prepareSearch("output2").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsString(), containsString("mytestresult"));
     }
 
@@ -212,13 +212,13 @@ public class TransformIntegrationTests extends AbstractWatcherIntegrationTestCas
 
         SearchResponse response = client().prepareSearch("output1").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1));
         assertThat(response.getHits().getAt(0).sourceAsMap().get("key4").toString(), equalTo("30"));
 
         response = client().prepareSearch("output2").get();
         assertNoFailures(response);
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1l));
+        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
         assertThat(response.getHits().getAt(0).sourceAsMap().size(), equalTo(1));
         assertThat(response.getHits().getAt(0).sourceAsMap().get("key4").toString(), equalTo("30"));
     }

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/BasicWatcherTests.java

@@ -88,7 +88,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1l))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L))
                         .addAction("_logger", loggingAction("\n\n************\n" +
                                         "total hits: {{ctx.payload.hits.total}}\n" +
                                         "************\n")
@@ -114,7 +114,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1l)))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
 
         if (timeWarped()) {
@@ -145,7 +145,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1l)))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
         assertThat(indexResponse.isCreated(), is(true));
 
@@ -162,7 +162,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
         assertThat(deleteWatchResponse.isFound(), is(true));
 
         refresh();
-        assertHitCount(client().prepareSearch(WatchStore.INDEX).setSize(0).get(), 0l);
+        assertHitCount(client().prepareSearch(WatchStore.INDEX).setSize(0).get(), 0L);
 
         // Deleting the same watch for the second time
         deleteWatchResponse = watcherClient.prepareDeleteWatch("_name").get();
@@ -214,7 +214,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .addAction("_id", indexAction("idx", "action"));
 
         watcherClient().preparePutWatch("_name")
-                .setSource(source.condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1l)))
+                .setSource(source.condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
 
         if (timeWarped()) {
@@ -225,7 +225,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
         assertWatchWithMinimumPerformedActionsCount("_name", 0, false);
 
         watcherClient().preparePutWatch("_name")
-                .setSource(source.condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0l)))
+                .setSource(source.condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
                 .get();
 
         if (timeWarped()) {
@@ -238,7 +238,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
         watcherClient().preparePutWatch("_name")
                 .setSource(source
                         .trigger(schedule(Schedules.cron("0/1 * * * * ? 2020")))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0l)))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
                 .get();
 
         if (timeWarped()) {
@@ -340,14 +340,14 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest).extractKeys("hits.total"))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1l)))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
         // in this watcher the condition will fail, because max_score isn't extracted, only total:
         watcherClient.preparePutWatch("_name2")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest).extractKeys("hits.total"))
-                        .condition(compareCondition("ctx.payload.hits.max_score", CompareCondition.Op.GTE, 0l)))
+                        .condition(compareCondition("ctx.payload.hits.max_score", CompareCondition.Op.GTE, 0L)))
                 .get();
 
         if (timeWarped()) {
@@ -454,7 +454,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(searchInput(request))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3l)))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3L)))
                 .get();
 
         logger.info("created watch [{}] at [{}]", watchName, SystemClock.INSTANCE.nowUTC());

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/EmailActionIntegrationTests.java

@@ -94,7 +94,7 @@ public class EmailActionIntegrationTests extends AbstractWatcherIntegrationTestC
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0l))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
                         .addAction("_email", emailAction(EmailTemplate.builder().from("_from").to("_to")
                                 .subject("{{ctx.payload.hits.hits.0._source.field}}")).setAuthentication(USERNAME, PASSWORD.toCharArray())))
                         .get();

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/EmailAttachmentTests.java

@@ -194,7 +194,7 @@ public class EmailAttachmentTests extends AbstractWatcherIntegrationTestCase {
         WatchSourceBuilder watchSourceBuilder = watchBuilder()
                 .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                 .input(searchInput(searchRequest))
-                .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0l))
+                .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
                 .addAction("_email", emailAction(emailBuilder).setAuthentication(USERNAME, PASSWORD.toCharArray())
                 .setAttachments(emailAttachments));
         logger.info("TMP WATCHSOURCE {}", watchSourceBuilder.build().getBytes().toUtf8());

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/TimeThrottleIntegrationTests.java

@@ -75,7 +75,7 @@ public class TimeThrottleIntegrationTests extends AbstractWatcherIntegrationTest
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(searchInput(matchAllRequest().indices("events")))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0l))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
                         .transform(searchTransform(matchAllRequest().indices("events")))
                         .addAction("_id", indexAction("actions", "action"))
                         .defaultThrottlePeriod(TimeValue.timeValueSeconds(30)))
@@ -149,7 +149,7 @@ public class TimeThrottleIntegrationTests extends AbstractWatcherIntegrationTest
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("1s")))
                         .input(searchInput(matchAllRequest().indices("events")))
-                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0l))
+                        .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GT, 0L))
                         .transform(searchTransform(matchAllRequest().indices("events")))
                         .addAction("_id", indexAction("actions", "action")))
                 .get();

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java

@@ -643,7 +643,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             @Override
             public boolean matches(long expirationDate, long now) {
                 long expiryDuration = expirationDate - now;
-                if (expiryDuration > 0l) {
+                if (expiryDuration > 0L) {
                     if (expiryDuration <= max.getMillis()) {
                         return expiryDuration >= min.getMillis();
                     }
@@ -673,7 +673,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             @Override
             public boolean matches(long expirationDate, long now) {
                 long postExpiryDuration = now - expirationDate;
-                if (postExpiryDuration > 0l) {
+                if (postExpiryDuration > 0L) {
                     if (postExpiryDuration <= max.getMillis()) {
                         return postExpiryDuration >= min.getMillis();
                     }
@@ -684,12 +684,12 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             @Override
             public TimeValue delay(long expiryDuration) {
                 final long delay;
-                if (expiryDuration >= 0l) {
+                if (expiryDuration >= 0L) {
                      delay = expiryDuration + min.getMillis();
                 } else {
-                     delay = (-1l * expiryDuration) - min.getMillis();
+                     delay = (-1L * expiryDuration) - min.getMillis();
                 }
-                if (delay > 0l) {
+                if (delay > 0L) {
                     return TimeValue.timeValueMillis(delay);
                 } else {
                     return null;

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesConsumerPluginIntegrationTestCase.java

@@ -126,7 +126,7 @@ public abstract class AbstractLicensesConsumerPluginIntegrationTestCase extends
         assertLicenseeState(consumerPlugin.id(), LicenseState.ENABLED);
 
         logger.info(" --> sleep for rest of trailLicense duration");
-        Thread.sleep(trialLicenseDurationInSeconds * 1000l);
+        Thread.sleep(trialLicenseDurationInSeconds * 1000L);
 
         logger.info(" --> check consumer is still enabled [signed license]");
         // consumer plugin should notify onEnabled on all data nodes (signed license)

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesPluginIntegrationTests.java

@@ -121,7 +121,7 @@ public class LicensesPluginIntegrationTests extends AbstractLicensesIntegrationT
         assertLicenseeState(getCurrentFeatureName(), LicenseState.ENABLED);
 
         logger.info(" --> sleep for rest of trailLicense duration");
-        Thread.sleep(trialLicenseDurationInSeconds * 1000l);
+        Thread.sleep(trialLicenseDurationInSeconds * 1000L);
 
         logger.info(" --> check consumer is still enabled [signed license]");
         // consumer plugin should notify onEnabled on all data nodes (signed license)

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java

@@ -78,7 +78,7 @@ public class LicensesTransportTests extends AbstractLicensesIntegrationTestCase
         // modify content of signed license
         License tamperedLicense = License.builder()
                 .fromLicenseSpec(signedLicense, signedLicense.signature())
-                .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000l)
+                .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000L)
                 .validate()
                 .build();
 

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java

@@ -117,7 +117,7 @@ public class TestUtils {
     }
 
     public static License generateSignedLicense(String type, long issueDate, TimeValue expiryDuration) throws Exception {
-        long issue = (issueDate != -1l) ? issueDate : System.currentTimeMillis();
+        long issue = (issueDate != -1L) ? issueDate : System.currentTimeMillis();
         int version = randomIntBetween(License.VERSION_START, License.VERSION_CURRENT);
         final String licenseType;
         if (version < License.VERSION_NO_FEATURE_TYPE) {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java

@@ -79,7 +79,7 @@ public class LicensesManagerServiceTests extends ESSingleNodeTestCase {
         // modify content of signed license
         License tamperedLicense = License.builder()
                 .fromLicenseSpec(signedLicense, signedLicense.signature())
-                .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000l)
+                .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000L)
                 .validate()
                 .build();
 

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/shield/authz/AuthorizationUtilsTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.SystemUser;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.InternalAuthenticationService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Unit tests for the AuthorizationUtils class
+ */
+public class AuthorizationUtilsTest extends ESTestCase {
+
+    private ThreadContext threadContext;
+
+    @Before
+    public void setupContext() {
+        threadContext = new ThreadContext(Settings.EMPTY);
+    }
+
+    public void testSystemUserSwitchNonInternalAction() {
+        assertThat(AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, randomFrom("indices:foo", "cluster:bar")), is(false));
+    }
+
+    public void testSystemUserSwitchWithNullorSystemUser() {
+        if (randomBoolean()) {
+            threadContext.putTransient(InternalAuthenticationService.USER_KEY, SystemUser.INSTANCE);
+        }
+        assertThat(AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, "internal:something"), is(true));
+    }
+
+    public void testSystemUserSwitchWithNonSystemUser() {
+        User user = new User(randomAsciiOfLength(6), new String[] {});
+        threadContext.putTransient(InternalAuthenticationService.USER_KEY, user);
+        threadContext.putTransient(InternalAuthorizationService.ORIGINATING_ACTION_KEY, randomFrom("indices:foo", "cluster:bar"));
+        assertThat(AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, "internal:something"), is(true));
+    }
+
+    public void testSystemUserSwitchWithNonSystemUserAndInternalAction() {
+        User user = new User(randomAsciiOfLength(6), new String[] {});
+        threadContext.putTransient(InternalAuthenticationService.USER_KEY, user);
+        threadContext.putTransient(InternalAuthorizationService.ORIGINATING_ACTION_KEY, randomFrom("internal:foo/bar"));
+        assertThat(AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, "internal:something"), is(false));
+    }
+}

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelPlugin.java

@@ -23,11 +23,8 @@ import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.cleaner.CleanerService;
 import org.elasticsearch.marvel.license.LicenseModule;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.marvel.shield.InternalMarvelUser;
-import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
 import org.elasticsearch.marvel.shield.MarvelShieldModule;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.shield.authz.AuthorizationModule;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.ArrayList;
@@ -126,16 +123,6 @@ public class MarvelPlugin extends Plugin {
         return false;
     }
 
-    // NOTE: The fact this signature takes a module is a hack, and effectively like the previous
-    // processModule in the plugin api. The problem is tight coupling between marvel and shield.
-    // We need to avoid trying to load the AuthorizationModule class unless we know shield integration
-    // is enabled. This is a temporary solution until inter-plugin-communication can be worked out.
-    public void onModule(Module module) {
-        if (MarvelShieldIntegration.enabled(settings) && module instanceof AuthorizationModule) {
-            ((AuthorizationModule)module).registerReservedRole(InternalMarvelUser.ROLE);
-        }
-    }
-
     public void onModule(SettingsModule module) {
         module.registerSetting(Exporters.EXPORTERS_SETTING);
         module.registerSetting(MarvelSettings.INDICES_SETTING);

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java

@@ -17,7 +17,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -40,8 +40,8 @@ public class ClusterStateCollector extends AbstractCollector<ClusterStateCollect
     private final Client client;
 
     @Inject
-    public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
+    public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
-                                 SecuredClient client) {
+                                 MarvelLicensee marvelLicensee, InternalClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
     }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollector.java

@@ -19,7 +19,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -49,7 +49,7 @@ public class ClusterStatsCollector extends AbstractCollector<ClusterStatsCollect
 
     @Inject
     public ClusterStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
-                                 SecuredClient client, LicensesManagerService licensesManagerService, ClusterName clusterName) {
+                                 InternalClient client, LicensesManagerService licensesManagerService, ClusterName clusterName) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
         this.clusterName = clusterName;

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java

@@ -18,7 +18,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -40,8 +40,8 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
     private final Client client;
 
     @Inject
-    public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
+    public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
-                                  SecuredClient client) {
+                                  MarvelLicensee marvelLicensee, InternalClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
     }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java

@@ -19,7 +19,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -41,8 +41,8 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
     private final Client client;
 
     @Inject
-    public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
+    public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
-                               SecuredClient client) {
+                               MarvelLicensee marvelLicensee, InternalClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
     }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java

@@ -18,7 +18,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -37,8 +37,8 @@ public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollect
     private final Client client;
 
     @Inject
-    public IndicesStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
+    public IndicesStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,
-                                 SecuredClient client) {
+                                 MarvelLicensee marvelLicensee, InternalClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
     }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollector.java

@@ -22,7 +22,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -48,7 +48,7 @@ public class NodeStatsCollector extends AbstractCollector<NodeStatsCollector> {
 
     @Inject
     public NodeStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
-                              SecuredClient client, DiscoveryService discoveryService, NodeEnvironment nodeEnvironment,
+                              InternalClient client, DiscoveryService discoveryService, NodeEnvironment nodeEnvironment,
                               DiskThresholdDecider diskThresholdDecider) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java

@@ -31,7 +31,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.cleaner.CleanerService;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
@@ -319,13 +319,13 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     public static class Factory extends Exporter.Factory<LocalExporter> {
 
-        private final SecuredClient client;
+        private final InternalClient client;
         private final RendererRegistry registry;
         private final ClusterService clusterService;
         private final CleanerService cleanerService;
 
         @Inject
-        public Factory(SecuredClient client, ClusterService clusterService, RendererRegistry registry, CleanerService cleanerService) {
+        public Factory(InternalClient client, ClusterService clusterService, RendererRegistry registry, CleanerService cleanerService) {
             super(TYPE, true);
             this.client = client;
             this.clusterService = clusterService;

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/license/LicenseModule.java

@@ -9,20 +9,9 @@ import org.elasticsearch.common.inject.AbstractModule;
 
 public class LicenseModule extends AbstractModule {
 
-    public LicenseModule() {
-        verifyLicensePlugin();
-    }
-
     @Override
     protected void configure() {
         bind(MarvelLicensee.class).asEagerSingleton();
     }
 
-    private void verifyLicensePlugin() {
-        try {
-            getClass().getClassLoader().loadClass("org.elasticsearch.license.plugin.LicensePlugin");
-        } catch (ClassNotFoundException cnfe) {
-            throw new IllegalStateException("marvel plugin requires the license plugin to be installed");
-        }
-    }
 }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/shield/InternalMarvelUser.java

@@ -1,47 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.marvel.shield;
-
-import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
-import org.elasticsearch.marvel.agent.settings.MarvelSettings;
-import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.authz.permission.Role;
-import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.shield.authz.privilege.IndexPrivilege;
-import org.elasticsearch.shield.authz.privilege.Privilege;
-
-/**
- *
- */
-public class InternalMarvelUser extends User {
-
-    static final String NAME = "__marvel_user";
-    static final String[] ROLE_NAMES = new String[] { "__marvel_role" };
-
-    public static final InternalMarvelUser INSTANCE = new InternalMarvelUser(NAME, ROLE_NAMES);
-
-    public static final Role ROLE = Role.builder(ROLE_NAMES[0])
-            .cluster(ClusterPrivilege.get(new Privilege.Name(
-                    PutIndexTemplateAction.NAME + "*",
-                    GetIndexTemplatesAction.NAME + "*",
-                    ClusterPrivilege.MONITOR.name().toString())))
-
-            // we need all monitoring access
-            .add(IndexPrivilege.MONITOR, "*")
-
-            // and full access to .marvel-es-* and .marvel-es-data indices
-            .add(IndexPrivilege.ALL, MarvelSettings.MARVEL_INDICES_PREFIX + "*")
-
-            // note, we don't need _license permission as we're taking the licenses
-            // directly form the license service.
-
-            .build();
-
-    InternalMarvelUser(String username, String[] roles) {
-        super(username, roles);
-    }
-}

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldIntegration.java

@@ -5,47 +5,23 @@
  */
 package org.elasticsearch.marvel.shield;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.ShieldSettingsFilter;
-import org.elasticsearch.shield.authc.AuthenticationService;
-
-import java.io.IOException;
 
 /**
  *
  */
 public class MarvelShieldIntegration {
 
-    private final boolean enabled;
-    private final AuthenticationService authcService;
     private final ShieldSettingsFilter settingsFilter;
-    private final Client client;
 
     @Inject
     public MarvelShieldIntegration(Settings settings, Injector injector) {
-        enabled = enabled(settings);
+        boolean enabled = enabled(settings);
-        authcService = enabled ? injector.getInstance(AuthenticationService.class) : null;
         settingsFilter = enabled ? injector.getInstance(ShieldSettingsFilter.class) : null;
-        client = injector.getInstance(Client.class);
-    }
-
-    public Client getClient() {
-        return client;
-    }
-
-    public void bindInternalMarvelUser() {
-        if (authcService != null) {
-            try {
-                authcService.attachUserHeaderIfMissing(InternalMarvelUser.INSTANCE);
-            } catch (IOException e) {
-                throw new ElasticsearchException("failed to attach marvel user to request", e);
-            }
-        }
     }
 
     public void filterOutSettings(String... patterns) {

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldModule.java

@@ -7,7 +7,6 @@ package org.elasticsearch.marvel.shield;
 
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.marvel.MarvelPlugin;
 
 /**
  *
@@ -15,19 +14,14 @@ import org.elasticsearch.marvel.MarvelPlugin;
 public class MarvelShieldModule extends AbstractModule {
 
     private final boolean shieldEnabled;
-    private final boolean marvelEnabled;
 
     public MarvelShieldModule(Settings settings) {
         this.shieldEnabled = MarvelShieldIntegration.enabled(settings);
-        this.marvelEnabled = MarvelPlugin.marvelEnabled(settings);;
     }
 
     @Override
     protected void configure() {
         bind(MarvelShieldIntegration.class).asEagerSingleton();
-        if (marvelEnabled) {
-            bind(SecuredClient.class).asEagerSingleton();
-        }
         if (shieldEnabled) {
             bind(MarvelSettingsFilter.Shield.class).asEagerSingleton();
             bind(MarvelSettingsFilter.class).to(MarvelSettingsFilter.Shield.class);

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/shield/SecuredClient.java

@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.marvel.shield;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.FilterClient;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-
-/**
- *
- */
-public class SecuredClient extends FilterClient {
-
-    private MarvelShieldIntegration shieldIntegration;
-
-    @Inject
-    public SecuredClient(Client in, MarvelShieldIntegration shieldIntegration) {
-        super(in);
-        this.shieldIntegration = shieldIntegration;
-    }
-
-    @Override
-    protected <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
-                Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
-        try (ThreadContext.StoredContext ctx = threadPool().getThreadContext().stashContext()) {
-            this.shieldIntegration.bindInternalMarvelUser();
-            super.doExecute(action, request, listener);
-        }
-    }
-}

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelF.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.marvel;
 
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -12,6 +14,7 @@ import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.XPackPlugin;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 
@@ -42,9 +45,14 @@ public class MarvelF {
 
             @Override
             public void run() {
-                node.close();
+                try {
+                    IOUtils.close(node);
+                } catch (IOException e) {
+                    throw new ElasticsearchException(e);
+                } finally {
                     latch.countDown();
                 }
+            }
         });
         node.start();
         latch.await();

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java

@@ -24,11 +24,9 @@ import org.elasticsearch.license.plugin.core.Licensee;
 import org.elasticsearch.license.plugin.core.LicenseeRegistry;
 import org.elasticsearch.license.plugin.core.LicensesManagerService;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
-import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
-import org.elasticsearch.marvel.shield.SecuredClient;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.shield.InternalClient;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.xpack.XPackPlugin;
@@ -64,16 +62,12 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase {
         enableLicense();
     }
 
-    public SecuredClient securedClient() {
+    public InternalClient securedClient() {
-        MarvelShieldIntegration integration = internalCluster().getInstance(MarvelShieldIntegration.class);
+        return internalCluster().getInstance(InternalClient.class);
-        // we must get the client from the same node!
-        return new SecuredClient(integration.getClient(), integration);
     }
 
-    public SecuredClient securedClient(String nodeId) {
+    public InternalClient securedClient(String nodeId) {
-        MarvelShieldIntegration integration = internalCluster().getInstance(MarvelShieldIntegration.class, nodeId);
+        return internalCluster().getInstance(InternalClient.class, nodeId);
-        // we must get the client from the same node!
-        return new SecuredClient(integration.getClient(), integration);
     }
 
     protected void assertCanCollect(AbstractCollector collector) {

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java

@@ -116,7 +116,8 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
 
             for (RecoveryState shardRecovery : shardRecoveries) {
                 assertThat(shard.getKey(), equalTo(indexName));
-                assertThat(shardRecovery.getType(), anyOf(equalTo(RecoveryState.Type.RELOCATION), equalTo(RecoveryState.Type.STORE), equalTo(RecoveryState.Type.REPLICA)));
+                assertThat(shardRecovery.getType(), anyOf(equalTo(RecoveryState.Type.PRIMARY_RELOCATION), equalTo(RecoveryState.Type.STORE),
+                        equalTo(RecoveryState.Type.REPLICA), equalTo(RecoveryState.Type.SNAPSHOT)));
             }
         }
     }

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/node/NodeStatsCollectorTests.java

@@ -15,7 +15,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.shield.InternalClient;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
 import java.util.Collection;
@@ -91,7 +91,7 @@ public class NodeStatsCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(ClusterService.class, nodeId),
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(MarvelLicensee.class, nodeId),
-                internalCluster().getInstance(SecuredClient.class, nodeId),
+                internalCluster().getInstance(InternalClient.class, nodeId),
                 internalCluster().getInstance(DiscoveryService.class, nodeId),
                 internalCluster().getInstance(NodeEnvironment.class, nodeId),
                 internalCluster().getInstance(DiskThresholdDecider.class, nodeId));

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java

@@ -16,8 +16,7 @@ import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.cleaner.CleanerService;
 import org.elasticsearch.marvel.shield.MarvelSettingsFilter;
-import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
+import org.elasticsearch.shield.InternalClient;
-import org.elasticsearch.marvel.shield.SecuredClient;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -60,7 +59,7 @@ public class ExportersTests extends ESTestCase {
         clusterService = mock(ClusterService.class);
 
         // we always need to have the local exporter as it serves as the default one
-        factories.put(LocalExporter.TYPE, new LocalExporter.Factory(new SecuredClient(client, mock(MarvelShieldIntegration.class)), clusterService, mock(RendererRegistry.class), mock(CleanerService.class)));
+        factories.put(LocalExporter.TYPE, new LocalExporter.Factory(new InternalClient.Insecure(client), clusterService, mock(RendererRegistry.class), mock(CleanerService.class)));
         clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(MarvelSettings.COLLECTORS_SETTING, MarvelSettings.INTERVAL_SETTING, Exporters.EXPORTERS_SETTING)));
         settingsFilter = mock(MarvelSettingsFilter.class);
         exporters = new Exporters(Settings.EMPTY, factories, settingsFilter, clusterService, clusterSettings);

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryRendererTests.java

@@ -35,7 +35,7 @@ public class IndexRecoveryRendererTests extends ESTestCase {
         List<RecoveryState> shards = new ArrayList<>();
 
         // Shard 0
-        RecoveryState shard0 = new RecoveryState(new ShardId(indexName, "testUUID", 0), true, RecoveryState.Type.RELOCATION, source, target);
+        RecoveryState shard0 = new RecoveryState(new ShardId(indexName, "testUUID", 0), true, RecoveryState.Type.PRIMARY_RELOCATION, source, target);
         shards.add(shard0);
 
         // Shard 1

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelInternalClientTests.java

@@ -13,13 +13,13 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.shield.InternalClient;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.is;
 
-public class SecuredClientTests extends MarvelIntegTestCase {
+public class MarvelInternalClientTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -31,37 +31,37 @@ public class SecuredClientTests extends MarvelIntegTestCase {
     }
 
     public void testAllowedAccess() {
-        SecuredClient securedClient = internalCluster().getInstance(SecuredClient.class);
+        InternalClient internalClient = internalCluster().getInstance(InternalClient.class);
 
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareHealth());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareHealth());
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareClusterStats());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareClusterStats());
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareState());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareState());
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareNodesInfo());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareNodesInfo());
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareNodesStats());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareNodesStats());
-        assertAccessIsAllowed(securedClient.admin().cluster().prepareNodesHotThreads());
+        assertAccessIsAllowed(internalClient.admin().cluster().prepareNodesHotThreads());
 
-        assertAccessIsAllowed(securedClient.admin().indices().prepareGetSettings());
+        assertAccessIsAllowed(internalClient.admin().indices().prepareGetSettings());
-        assertAccessIsAllowed(securedClient.admin().indices().prepareSegments());
+        assertAccessIsAllowed(internalClient.admin().indices().prepareSegments());
-        assertAccessIsAllowed(securedClient.admin().indices().prepareRecoveries());
+        assertAccessIsAllowed(internalClient.admin().indices().prepareRecoveries());
-        assertAccessIsAllowed(securedClient.admin().indices().prepareStats());
+        assertAccessIsAllowed(internalClient.admin().indices().prepareStats());
 
-        assertAccessIsAllowed(securedClient.admin().indices().prepareDelete(MarvelSettings.MARVEL_INDICES_PREFIX));
+        assertAccessIsAllowed(internalClient.admin().indices().prepareDelete(MarvelSettings.MARVEL_INDICES_PREFIX));
-        assertAccessIsAllowed(securedClient.admin().indices().prepareCreate(MarvelSettings.MARVEL_INDICES_PREFIX + "test"));
+        assertAccessIsAllowed(internalClient.admin().indices().prepareCreate(MarvelSettings.MARVEL_INDICES_PREFIX + "test"));
 
-        assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
+        assertAccessIsAllowed(internalClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
-        assertAccessIsAllowed(securedClient.admin().indices().prepareGetTemplates("foo"));
+        assertAccessIsAllowed(internalClient.admin().indices().prepareGetTemplates("foo"));
     }
 
     public void testDeniedAccess() {
-        SecuredClient securedClient = internalCluster().getInstance(SecuredClient.class);
+        InternalClient internalClient = internalCluster().getInstance(InternalClient.class);
-        assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
+        assertAcked(internalClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
 
         if (shieldEnabled) {
-            assertAccessIsDenied(securedClient.admin().indices().prepareDeleteTemplate("foo"));
+            assertAccessIsDenied(internalClient.admin().indices().prepareDeleteTemplate("foo"));
-            assertAccessIsDenied(securedClient.admin().cluster().prepareGetRepositories());
+            assertAccessIsDenied(internalClient.admin().cluster().prepareGetRepositories());
         } else {
-            assertAccessIsAllowed(securedClient.admin().indices().prepareDeleteTemplate("foo"));
+            assertAccessIsAllowed(internalClient.admin().indices().prepareDeleteTemplate("foo"));
-            assertAccessIsAllowed(securedClient.admin().cluster().prepareGetRepositories());
+            assertAccessIsAllowed(internalClient.admin().cluster().prepareGetRepositories());
         }
     }
 

elasticsearch/x-pack/shield/dev-tools/esvm/.esvmrc

@@ -12,7 +12,6 @@
       "discovery": {
         "type": "zen",
         "zen.ping" : {
-          "multicast.enabled": false,
           "unicast.hosts": [ "127.0.0.1:9300", "127.0.0.1:9301" ]
         }
       },

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/InternalClient.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.FilterClient;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.authc.AuthenticationService;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public abstract class InternalClient extends FilterClient {
+
+    protected InternalClient(Client in) {
+        super(in);
+    }
+
+    /**
+     * An insecured internal client, baseically simply delegates to the normal ES client
+     * without doing anything extra.
+     */
+    public static class Insecure extends InternalClient {
+
+        @Inject
+        public Insecure(Client in) {
+            super(in);
+        }
+    }
+
+    /**
+     * A secured internal client that binds the internal XPack user to the current
+     * execution context, before the action is executed.
+     */
+    public static class Secure extends InternalClient {
+
+        private AuthenticationService authcService;
+
+        @Inject
+        public Secure(Client in, AuthenticationService authcService) {
+            super(in);
+            this.authcService = authcService;
+        }
+
+        @Override
+        protected <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
+                Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+
+            try (ThreadContext.StoredContext ctx = threadPool().getThreadContext().stashContext()) {
+                try {
+                    authcService.attachUserHeaderIfMissing(XPackUser.INSTANCE);
+                } catch (IOException ioe) {
+                    throw new ElasticsearchException("failed to attach internal user to request", ioe);
+                }
+                super.doExecute(action, request, listener);
+            }
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/SecurityContext.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+/**
+ *
+ */
+public interface SecurityContext {
+
+    void executeAs(User user, Runnable runnable);
+
+    <V> V executeAs(User user, Callable<V> callable);
+
+    class Insecure implements SecurityContext {
+
+        public static final Insecure INSTANCE = new Insecure();
+
+        private Insecure() {
+        }
+
+        @Override
+        public void executeAs(User user, Runnable runnable) {
+            runnable.run();
+        }
+
+        @Override
+        public <V> V executeAs(User user, Callable<V> callable) {
+            try {
+                return callable.call();
+            } catch (Exception e) {
+                throw new ElasticsearchException(e);
+            }
+        }
+    }
+
+    class Secure implements SecurityContext {
+
+        private final ThreadContext threadContext;
+        private final AuthenticationService authcService;
+
+        @Inject
+        public Secure(ThreadPool threadPool, AuthenticationService authcService) {
+            this.threadContext = threadPool.getThreadContext();
+            this.authcService = authcService;
+        }
+
+        public void executeAs(User user, Runnable runnable) {
+            try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+                setUser(user);
+                runnable.run();
+            }
+        }
+
+        public <V> V executeAs(User user, Callable<V> callable) {
+            try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+                setUser(user);
+                return callable.call();
+            } catch (Exception e) {
+                throw new ElasticsearchException(e);
+            }
+        }
+
+        private void setUser(User user) {
+            try {
+                authcService.attachUserHeaderIfMissing(user);
+            } catch (IOException e) {
+                throw new ElasticsearchException("failed to attach watcher user to request", e);
+            }
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldDisabledModule.java

@@ -22,6 +22,11 @@ public class ShieldDisabledModule extends AbstractShieldModule {
         if (!clientMode) {
             // required by the shield info rest action (when shield is disabled)
             bind(ShieldLicenseState.class).toProvider(Providers.<ShieldLicenseState>of(null));
+
+            bind(SecurityContext.class).toInstance(SecurityContext.Insecure.INSTANCE);
+
+            bind(InternalClient.Insecure.class).asEagerSingleton();
+            bind(InternalClient.class).to(InternalClient.Insecure.class);
         }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldModule.java

@@ -20,8 +20,14 @@ public class ShieldModule extends AbstractShieldModule {
     @Override
     protected void configure(boolean clientMode) {
         if (!clientMode) {
+            bind(SecurityContext.Secure.class).asEagerSingleton();
+            bind(SecurityContext.class).to(SecurityContext.Secure.class);
             bind(ShieldLifecycleService.class).asEagerSingleton();
             bind(ShieldSettingsFilter.class).asEagerSingleton();
+            bind(ShieldTemplateService.class).asEagerSingleton();
+
+            bind(InternalClient.Secure.class).asEagerSingleton();
+            bind(InternalClient.class).to(InternalClient.Secure.class);
         }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java

@@ -18,32 +18,23 @@ import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.action.ShieldActionFilter;
 import org.elasticsearch.shield.action.ShieldActionModule;
-import org.elasticsearch.shield.action.admin.role.AddRoleAction;
+import org.elasticsearch.shield.action.realm.ClearRealmCacheAction;
-import org.elasticsearch.shield.action.admin.role.DeleteRoleAction;
+import org.elasticsearch.shield.action.realm.TransportClearRealmCacheAction;
-import org.elasticsearch.shield.action.admin.role.GetRolesAction;
+import org.elasticsearch.shield.action.role.AddRoleAction;
-import org.elasticsearch.shield.action.admin.role.RestAddRoleAction;
+import org.elasticsearch.shield.action.role.ClearRolesCacheAction;
-import org.elasticsearch.shield.action.admin.role.RestDeleteRoleAction;
+import org.elasticsearch.shield.action.role.DeleteRoleAction;
-import org.elasticsearch.shield.action.admin.role.RestGetRolesAction;
+import org.elasticsearch.shield.action.role.GetRolesAction;
-import org.elasticsearch.shield.action.admin.role.TransportAddRoleAction;
+import org.elasticsearch.shield.action.role.TransportAddRoleAction;
-import org.elasticsearch.shield.action.admin.role.TransportDeleteRoleAction;
+import org.elasticsearch.shield.action.role.TransportClearRolesCacheAction;
-import org.elasticsearch.shield.action.admin.role.TransportGetRolesAction;
+import org.elasticsearch.shield.action.role.TransportDeleteRoleAction;
-import org.elasticsearch.shield.action.admin.user.AddUserAction;
+import org.elasticsearch.shield.action.role.TransportGetRolesAction;
-import org.elasticsearch.shield.action.admin.user.DeleteUserAction;
+import org.elasticsearch.shield.action.user.AddUserAction;
-import org.elasticsearch.shield.action.admin.user.GetUsersAction;
+import org.elasticsearch.shield.action.user.DeleteUserAction;
-import org.elasticsearch.shield.action.admin.user.RestAddUserAction;
+import org.elasticsearch.shield.action.user.GetUsersAction;
-import org.elasticsearch.shield.action.admin.user.RestDeleteUserAction;
+import org.elasticsearch.shield.action.user.TransportAddUserAction;
-import org.elasticsearch.shield.action.admin.user.RestGetUsersAction;
+import org.elasticsearch.shield.action.user.TransportDeleteUserAction;
-import org.elasticsearch.shield.action.admin.user.TransportAddUserAction;
+import org.elasticsearch.shield.action.user.TransportGetUsersAction;
-import org.elasticsearch.shield.action.admin.user.TransportDeleteUserAction;
-import org.elasticsearch.shield.action.admin.user.TransportGetUsersAction;
-import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheAction;
-import org.elasticsearch.shield.action.authc.cache.TransportClearRealmCacheAction;
-import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheAction;
-import org.elasticsearch.shield.action.authz.cache.TransportClearRolesCacheAction;
-import org.elasticsearch.shield.admin.ShieldAdminModule;
-import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
 import org.elasticsearch.shield.audit.AuditTrailModule;
-import org.elasticsearch.shield.audit.index.IndexAuditUserHolder;
 import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationModule;
 import org.elasticsearch.shield.authc.Realms;
@@ -61,8 +52,14 @@ import org.elasticsearch.shield.license.ShieldLicensee;
 import org.elasticsearch.shield.rest.ShieldRestModule;
 import org.elasticsearch.shield.rest.action.RestAuthenticateAction;
 import org.elasticsearch.shield.rest.action.RestShieldInfoAction;
-import org.elasticsearch.shield.rest.action.authc.cache.RestClearRealmCacheAction;
+import org.elasticsearch.shield.rest.action.realm.RestClearRealmCacheAction;
-import org.elasticsearch.shield.rest.action.authz.cache.RestClearRolesCacheAction;
+import org.elasticsearch.shield.rest.action.role.RestAddRoleAction;
+import org.elasticsearch.shield.rest.action.role.RestClearRolesCacheAction;
+import org.elasticsearch.shield.rest.action.role.RestDeleteRoleAction;
+import org.elasticsearch.shield.rest.action.role.RestGetRolesAction;
+import org.elasticsearch.shield.rest.action.user.RestAddUserAction;
+import org.elasticsearch.shield.rest.action.user.RestDeleteUserAction;
+import org.elasticsearch.shield.rest.action.user.RestGetUsersAction;
 import org.elasticsearch.shield.ssl.SSLModule;
 import org.elasticsearch.shield.transport.ShieldClientTransportService;
 import org.elasticsearch.shield.transport.ShieldServerTransportService;
@@ -118,13 +115,17 @@ public class ShieldPlugin extends Plugin {
 
     @Override
     public Collection<Module> nodeModules() {
-        if (enabled == false) {
+
-            return Collections.<Module>singletonList(new ShieldDisabledModule(settings));
+        if (!enabled) {
-        } else if (clientMode) {
+            return Collections.singletonList(new ShieldDisabledModule(settings));
+        }
+
+        if (clientMode) {
             return Arrays.<Module>asList(
                     new ShieldTransportModule(settings),
                     new SSLModule(settings));
-        } else {
+        }
+
         // we can't load that at construction time since the license plugin might not have been loaded at that point
         // which might not be the case during Plugin class instantiation. Once nodeModules are pulled
         // everything should have been loaded
@@ -139,10 +140,8 @@ public class ShieldPlugin extends Plugin {
                 new ShieldRestModule(settings),
                 new ShieldActionModule(settings),
                 new ShieldTransportModule(settings),
-                    new ShieldAdminModule(settings),
                 new SSLModule(settings));
     }
-    }
 
     @Override
     public Collection<Class<? extends LifecycleComponent>> nodeServices() {
@@ -254,15 +253,6 @@ public class ShieldPlugin extends Plugin {
         }
     }
 
-    public void onModule(AuthorizationModule module) {
-        if (enabled) {
-            module.registerReservedRole(ShieldInternalUserHolder.ROLE);
-            if (AuditTrailModule.auditingEnabled(settings)) {
-                module.registerReservedRole(IndexAuditUserHolder.ROLE);
-            }
-        }
-    }
-
     private void addUserSettings(Settings.Builder settingsBuilder) {
         String authHeaderSettingName = ThreadContext.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER;
         if (settings.get(authHeaderSettingName) != null) {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldTemplateService.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.admin;
+package org.elasticsearch.shield;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
@@ -22,8 +22,6 @@ import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.gateway.GatewayService;
-import org.elasticsearch.shield.authc.AuthenticationService;
-import org.elasticsearch.shield.support.ClientWithUser;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.ByteArrayOutputStream;
@@ -40,26 +38,20 @@ public class ShieldTemplateService extends AbstractComponent implements ClusterS
     public static final String SHIELD_TEMPLATE_NAME = "shield-index-template";
 
     private final ThreadPool threadPool;
-    private final Provider<Client> clientProvider;
+    private final Provider<InternalClient> clientProvider;
-    private final Provider<AuthenticationService> authProvider;
-    private final ShieldInternalUserHolder adminUser;
     private final AtomicBoolean templateCreationPending = new AtomicBoolean(false);
 
     @Inject
     public ShieldTemplateService(Settings settings, ClusterService clusterService,
-            Provider<Client> clientProvider, ThreadPool threadPool,
+            Provider<InternalClient> clientProvider, ThreadPool threadPool) {
-            Provider<AuthenticationService> authProvider,
-            ShieldInternalUserHolder userHolder) {
         super(settings);
         this.threadPool = threadPool;
         this.clientProvider = clientProvider;
-        this.authProvider = authProvider;
-        this.adminUser = userHolder;
         clusterService.add(this);
     }
 
     private void createShieldTemplate() {
-        final Client client = getClient();
+        final Client client = clientProvider.get();
         try (InputStream is = getClass().getResourceAsStream("/" + SHIELD_TEMPLATE_NAME + ".json")) {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             Streams.copy(is, out);
@@ -77,11 +69,6 @@ public class ShieldTemplateService extends AbstractComponent implements ClusterS
             throw new IllegalStateException("failed to create shield admin index template [" +
                     SHIELD_ADMIN_INDEX_NAME + "]", e);
         }
-
-    }
-
-    Client getClient() {
-        return new ClientWithUser(clientProvider.get(), authProvider.get(), adminUser.user());
     }
 
     @Override

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/SystemUser.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.shield.authz.privilege.SystemPrivilege;
+
+import java.util.function.Predicate;
+
+/**
+ * Shield internal user that manages the {@code .shield}
+ * index. Has permission to monitor the cluster as well as all actions that deal
+ * with the shield admin index.
+ */
+public class SystemUser extends User {
+
+    public static final String NAME = "__es_system_user";
+    public static final String ROLE_NAME = "__es_system_role";
+
+    public static final User INSTANCE = new SystemUser();
+
+    private static final Predicate<String> PREDICATE = SystemPrivilege.INSTANCE.predicate();
+
+    private SystemUser() {
+        super(NAME, ROLE_NAME);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o == INSTANCE;
+    }
+
+    public static boolean is(User user) {
+        return INSTANCE.equals(user);
+    }
+
+    public static boolean isAuthorized(String action) {
+        return PREDICATE.test(action);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/User.java

@@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.shield.authz.SystemRole;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -21,8 +20,6 @@ import java.util.Arrays;
  */
 public class User implements ToXContent {
 
-    public static final User SYSTEM = new System();
-
     private final String username;
     private final String[] roles;
     private final User runAs;
@@ -37,7 +34,7 @@ public class User implements ToXContent {
         this.username = username;
         this.roles = roles == null ? Strings.EMPTY_ARRAY : roles;
         assert (runAs == null || runAs.runAs() == null) : "the runAs user should not be a user that can run as";
-        if (runAs == SYSTEM) {
+        if (runAs == SystemUser.INSTANCE) {
             throw new ElasticsearchSecurityException("the runAs user cannot be the internal system user");
         }
         this.runAs = runAs;
@@ -69,10 +66,6 @@ public class User implements ToXContent {
         return runAs;
     }
 
-    public final boolean isSystem() {
-        return this == SYSTEM;
-    }
-
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
@@ -91,43 +84,6 @@ public class User implements ToXContent {
         return sb.toString();
     }
 
-    public static User readFrom(StreamInput input) throws IOException {
-        if (input.readBoolean()) {
-            String name = input.readString();
-            if (System.NAME.equals(name)) {
-                return SYSTEM;
-            } else {
-                throw new IllegalStateException("invalid system user");
-            }
-        }
-        String username = input.readString();
-        String[] roles = input.readStringArray();
-        if (input.readBoolean()) {
-            String runAsUsername = input.readString();
-            String[] runAsRoles = input.readStringArray();
-            return new User(username, roles, new User(runAsUsername, runAsRoles));
-        }
-        return new User(username, roles);
-    }
-
-    public static void writeTo(User user, StreamOutput output) throws IOException {
-        if (user.isSystem()) {
-            output.writeBoolean(true);
-            output.writeString(System.NAME);
-        } else {
-            output.writeBoolean(false);
-            output.writeString(user.principal());
-            output.writeStringArray(user.roles());
-            if (user.runAs == null) {
-                output.writeBoolean(false);
-            } else {
-                output.writeBoolean(true);
-                output.writeString(user.runAs.principal());
-                output.writeStringArray(user.runAs.roles());
-            }
-        }
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -140,7 +96,6 @@ public class User implements ToXContent {
         if (runAs != null ? !runAs.equals(user.runAs) : user.runAs != null) {
             return false;
         }
-
         return true;
     }
 
@@ -161,12 +116,47 @@ public class User implements ToXContent {
         return builder;
     }
 
-    private static class System extends User {
+    public static User readFrom(StreamInput input) throws IOException {
-        private static final String NAME = "__es_system_user";
+        if (input.readBoolean()) {
-        private static final String[] ROLES = new String[] { SystemRole.NAME };
+            String name = input.readString();
+            switch (name) {
+                case SystemUser.NAME:
+                    return SystemUser.INSTANCE;
+                case XPackUser.NAME:
+                    return XPackUser.INSTANCE;
+                default:
+                    throw new IllegalStateException("invalid internal user");
+            }
+        }
+        String username = input.readString();
+        String[] roles = input.readStringArray();
+        if (input.readBoolean()) {
+            String runAsUsername = input.readString();
+            String[] runAsRoles = input.readStringArray();
+            return new User(username, roles, new User(runAsUsername, runAsRoles));
+        }
+        return new User(username, roles);
+    }
 
-        private System() {
+    public static void writeTo(User user, StreamOutput output) throws IOException {
-            super(NAME, ROLES);
+        if (SystemUser.is(user)) {
+            output.writeBoolean(true);
+            output.writeString(SystemUser.NAME);
+        } if (XPackUser.is(user)) {
+            output.writeBoolean(true);
+            output.writeString(XPackUser.NAME);
+        } else {
+            output.writeBoolean(false);
+            output.writeString(user.principal());
+            output.writeStringArray(user.roles());
+            if (user.runAs == null) {
+                output.writeBoolean(false);
+            } else {
+                output.writeBoolean(true);
+                output.writeString(user.runAs.principal());
+                output.writeStringArray(user.runAs.roles());
             }
         }
     }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/XPackUser.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
+import org.elasticsearch.shield.action.realm.ClearRealmCacheAction;
+import org.elasticsearch.shield.action.role.ClearRolesCacheAction;
+import org.elasticsearch.shield.authz.permission.Role;
+import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.shield.authz.privilege.IndexPrivilege;
+import org.elasticsearch.shield.authz.privilege.Privilege;
+
+/**
+ * XPack internal user that manages xpack. Has all cluster/indices permissions for watcher,
+ * shield and marvel to operate.
+ */
+public class XPackUser extends User {
+
+    public static final String NAME = "__es_internal_user";
+
+    public static final Role ROLE = Role.builder("__es_internal_role")
+            .cluster(ClusterPrivilege.get(new Privilege.Name(
+                    ClearRealmCacheAction.NAME + "*",       // shield
+                    ClearRolesCacheAction.NAME + "*",       // shield
+                    PutIndexTemplateAction.NAME,            // shield, marvel, watcher
+                    GetIndexTemplatesAction.NAME + "*",     // marvel
+                    ClusterPrivilege.MONITOR.name().toString())))  // marvel
+
+
+            // for now, the watches will be executed under the watcher user, meaning, all actions
+            // taken as part of the execution will be executed on behalf of this user. this includes
+            // the index action, search input and search transform. For this reason the watcher user
+            // requires full access to all indices in the cluster.
+            //
+            // at later phases we'll want to execute the watch on behalf of the user who registers
+            // it. this will require some work to attach/persist that user to/with the watch.
+            .add(IndexPrivilege.ALL, "*")
+
+
+//            these will be the index permissions required by shield (will uncomment once we optimize watcher permissions)
+
+//            .add(IndexPrivilege.ALL, ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+//            .add(IndexPrivilege.ALL, IndexAuditTrail.INDEX_NAME_PREFIX + "*")
+
+
+
+//            these will be the index permissions required by marvel (will uncomment once we optimize watcher permissions)
+
+//            // we need all monitoring access
+//            .add(IndexPrivilege.MONITOR, "*")
+//            // and full access to .marvel-es-* and .marvel-es-data indices
+//            .add(IndexPrivilege.ALL, MarvelSettings.MARVEL_INDICES_PREFIX + "*")
+
+            .build();
+
+    public static final XPackUser INSTANCE = new XPackUser();
+
+    XPackUser() {
+        super(NAME, ROLE.name());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return INSTANCE == o;
+    }
+
+    public static boolean is(User user) {
+        return INSTANCE.equals(user);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/ShieldActionFilter.java

@@ -20,12 +20,14 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.plugin.core.LicenseUtils;
 import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.shield.SystemUser;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.action.interceptor.RequestInterceptor;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.shield.authc.InternalAuthenticationService;
 import org.elasticsearch.shield.authz.AuthorizationService;
+import org.elasticsearch.shield.authz.AuthorizationUtils;
 import org.elasticsearch.shield.authz.privilege.HealthAndStatsPrivilege;
 import org.elasticsearch.shield.crypto.CryptoService;
 import org.elasticsearch.shield.license.ShieldLicenseState;
@@ -96,40 +98,10 @@ public class ShieldActionFilter extends AbstractComponent implements ActionFilte
                 threadContext.getTransient(InternalAuthenticationService.USER_KEY) != null;
         try {
             if (licenseState.securityEnabled()) {
-                // FIXME yet another hack. Needed to work around something like
+                if (AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, action)) {
-                /*
-                FailedNodeException[total failure in fetching]; nested: ElasticsearchSecurityException[action [internal:gateway/local/started_shards] is unauthorized for user [test_user]];
-                    at org.elasticsearch.gateway.AsyncShardFetch$1.onFailure(AsyncShardFetch.java:284)
-                    at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:84)
-                    at org.elasticsearch.shield.action.ShieldActionFilter.apply(ShieldActionFilter.java:121)
-                    at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:133)
-                    at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:107)
-                    at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:74)
-                    at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.list(TransportNodesListGatewayStartedShards.java:78)
-                    at org.elasticsearch.gateway.AsyncShardFetch.asyncFetch(AsyncShardFetch.java:274)
-                    at org.elasticsearch.gateway.AsyncShardFetch.fetchData(AsyncShardFetch.java:124)
-                    at org.elasticsearch.gateway.GatewayAllocator$InternalPrimaryShardAllocator.fetchData(GatewayAllocator.java:156)
-                    at org.elasticsearch.gateway.PrimaryShardAllocator.allocateUnassigned(PrimaryShardAllocator.java:83)
-                    at org.elasticsearch.gateway.GatewayAllocator.allocateUnassigned(GatewayAllocator.java:120)
-                    at org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators.allocateUnassigned(ShardsAllocators.java:72)
-                    at org.elasticsearch.cluster.routing.allocation.AllocationService.reroute(AllocationService.java:309)
-                    at org.elasticsearch.cluster.routing.allocation.AllocationService.reroute(AllocationService.java:273)
-                    at org.elasticsearch.cluster.routing.allocation.AllocationService.reroute(AllocationService.java:259)
-                    at org.elasticsearch.cluster.routing.RoutingService$2.execute(RoutingService.java:158)
-                    at org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:45)
-                    at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:447)
-                    at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:757)
-                    at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor$FilterRunnable.run(EsThreadPoolExecutor.java:211)
-                    at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:237)
-                    at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:200)
-                    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
-                    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
-                    at java.lang.Thread.run(Thread.java:745)
-                 */
-                if (INTERNAL_PREDICATE.test(action)) {
                     try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
                         String shieldAction = actionMapper.action(action, request);
-                        User user = authcService.authenticate(shieldAction, request, User.SYSTEM);
+                        User user = authcService.authenticate(shieldAction, request, SystemUser.INSTANCE);
                         authzService.authorize(user, shieldAction, request);
                         request = unsign(user, shieldAction, request);
 
@@ -144,7 +116,6 @@ public class ShieldActionFilter extends AbstractComponent implements ActionFilte
                     }
                 }
 
-
                 /**
                  here we fallback on the system user. Internal system requests are requests that are triggered by
                  the system itself (e.g. pings, update mappings, share relocation, etc...) and were not originated
@@ -155,9 +126,8 @@ public class ShieldActionFilter extends AbstractComponent implements ActionFilte
                  the {@link Rest} filter and the {@link ServerTransport} filter respectively), it's safe to assume a system user
                  here if a request is not associated with any other user.
                  */
-
                 String shieldAction = actionMapper.action(action, request);
-                User user = authcService.authenticate(shieldAction, request, User.SYSTEM);
+                User user = authcService.authenticate(shieldAction, request, SystemUser.INSTANCE);
                 authzService.authorize(user, shieldAction, request);
                 request = unsign(user, shieldAction, request);
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authc.cache;
+package org.elasticsearch.shield.action.realm;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authc.cache;
+package org.elasticsearch.shield.action.realm;
 
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authc.cache;
+package org.elasticsearch.shield.action.realm;
 
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/ClearRealmCacheResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authc.cache;
+package org.elasticsearch.shield.action.realm;
 
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/realm/TransportClearRealmCacheAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authc.cache;
+package org.elasticsearch.shield.action.realm;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/AddRoleAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/AddRoleRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/AddRoleRequestBuilder.java

@@ -3,17 +3,14 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.shield.authz.RoleDescriptor;
 
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
 
 /**
  * Builder for requests to add a role to the administrative index

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/AddRoleResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheAction.java

@@ -3,11 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authz.cache;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequestBuilder;
 
 /**
  * The action for clearing the cache used by native roles that are stored in an index.

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authz.cache;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authz.cache;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/ClearRolesCacheResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authz.cache;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/DeleteRoleResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/GetRolesResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/TransportAddRoleAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/TransportClearRolesCacheAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.authz.cache;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/TransportDeleteRoleAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/role/TransportGetRolesAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.role;
+package org.elasticsearch.shield.action.role;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -17,7 +17,6 @@ import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.Collections;
 import java.util.List;
 
 public class TransportGetRolesAction extends HandledTransportAction<GetRolesRequest, GetRolesResponse> {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/AddUserAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/AddUserRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequest;
@@ -12,7 +12,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.shield.authc.support.CharArrays;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/AddUserRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/AddUserResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/DeleteUserResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersRequest.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
-import java.util.Collections;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersRequestBuilder.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/GetUsersResponse.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/TransportAddUserAction.java

@@ -3,12 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/TransportDeleteUserAction.java

@@ -3,12 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/user/TransportGetUsersAction.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.shield.action.admin.user;
+package org.elasticsearch.shield.action.user;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -13,12 +13,10 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.authc.esnative.ESNativeRealm;
 import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.Collections;
 import java.util.List;
 
 public class TransportGetUsersAction extends HandledTransportAction<GetUsersRequest, GetUsersResponse> {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/admin/ShieldAdminModule.java

@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.admin;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.support.AbstractShieldModule;
-
-/**
- * TODO: document
- */
-public class ShieldAdminModule extends AbstractShieldModule.Node {
-
-    private ShieldInternalUserHolder userHolder;
-
-    public ShieldAdminModule(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    protected void configureNode() {
-        userHolder = new ShieldInternalUserHolder();
-        bind(ShieldInternalUserHolder.class).toInstance(userHolder);
-        bind(ShieldTemplateService.class).asEagerSingleton();
-    }
-}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/admin/ShieldInternalUserHolder.java

@@ -1,37 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.admin;
-
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
-import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.authz.permission.Role;
-import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.shield.authz.privilege.IndexPrivilege;
-import org.elasticsearch.shield.authz.privilege.Privilege;
-
-/**
- * User holder for the shield internal user that manages the {@code .shield}
- * index. Has permission to monitor the cluster as well as all actions that deal
- * with the shield admin index.
- */
-public class ShieldInternalUserHolder {
-
-    private static final String NAME = "__es_internal_user";
-    private static final String[] ROLES = new String[] { "__es_internal_role" };
-    public static final Role ROLE = Role.builder(ROLES[0])
-            .cluster(ClusterPrivilege.get(new Privilege.Name(PutIndexTemplateAction.NAME, "cluster:admin/shield/realm/cache/clear*", "cluster:admin/shield/roles/cache/clear*")))
-            .add(IndexPrivilege.ALL, ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
-            .build();
-    private static final User SHIELD_INTERNAL_USER = new User(NAME, ROLES);
-
-    public User user() {
-        return SHIELD_INTERNAL_USER;
-    }
-
-    public static boolean isShieldInternalUser(User user) {
-        return SHIELD_INTERNAL_USER.equals(user);
-    }
-}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java

@@ -9,9 +9,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.shield.ShieldLifecycleService;
 import org.elasticsearch.shield.audit.index.IndexAuditTrail;
-import org.elasticsearch.shield.audit.index.IndexAuditUserHolder;
 import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail;
 import org.elasticsearch.shield.support.AbstractShieldModule;
 
@@ -24,8 +22,6 @@ public class AuditTrailModule extends AbstractShieldModule.Node {
 
     private final boolean enabled;
 
-    private IndexAuditUserHolder indexAuditUser;
-
     public AuditTrailModule(Settings settings) {
         super(settings);
         enabled = auditingEnabled(settings);
@@ -37,7 +33,6 @@ public class AuditTrailModule extends AbstractShieldModule.Node {
             bind(AuditTrail.class).toInstance(AuditTrail.NOOP);
             return;
         }
-        indexAuditUser = new IndexAuditUserHolder();
         String[] outputs = settings.getAsArray("shield.audit.outputs", new String[] { LoggingAuditTrail.NAME });
         if (outputs.length == 0) {
             bind(AuditTrail.class).toInstance(AuditTrail.NOOP);
@@ -54,7 +49,6 @@ public class AuditTrailModule extends AbstractShieldModule.Node {
                     bind(LoggingAuditTrail.class).asEagerSingleton();
                     break;
                 case IndexAuditTrail.NAME:
-                    bind(IndexAuditUserHolder.class).toInstance(indexAuditUser);
                     binder.addBinding().to(IndexAuditTrail.class);
                     bind(IndexAuditTrail.class).asEagerSingleton();
                     break;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java

@@ -43,19 +43,20 @@ import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
+import org.elasticsearch.shield.InternalClient;
-import org.elasticsearch.shield.authz.privilege.SystemPrivilege;
+import org.elasticsearch.shield.SystemUser;
-import org.elasticsearch.shield.support.ClientWithUser;
-import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.XPackUser;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.shield.authc.AuthenticationToken;
+import org.elasticsearch.shield.authz.privilege.SystemPrivilege;
 import org.elasticsearch.shield.rest.RemoteHostHeader;
 import org.elasticsearch.shield.transport.filter.ShieldIpFilterRule;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.xpack.XPackPlugin;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
@@ -126,8 +127,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final String nodeName;
-    private final IndexAuditUserHolder auditUser;
+    private final Provider<InternalClient> clientProvider;
-    private final Provider<Client> clientProvider;
     private final AuthenticationService authenticationService;
     private final LinkedBlockingQueue<Message> eventQueue;
     private final QueueConsumer queueConsumer;
@@ -150,11 +150,9 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     }
 
     @Inject
-    public IndexAuditTrail(Settings settings, IndexAuditUserHolder indexingAuditUser,
+    public IndexAuditTrail(Settings settings, AuthenticationService authenticationService, Transport transport,
-                           AuthenticationService authenticationService, Transport transport,
+                           Provider<InternalClient> clientProvider, ThreadPool threadPool, ClusterService clusterService) {
-                           Provider<Client> clientProvider, ThreadPool threadPool, ClusterService clusterService) {
         super(settings);
-        this.auditUser = indexingAuditUser;
         this.authenticationService = authenticationService;
         this.clientProvider = clientProvider;
         this.transport = transport;
@@ -418,7 +416,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     public void accessGranted(User user, String action, TransportMessage<?> message) {
         if (!principalIsAuditor(user.principal())) {
             // special treatment for internal system actions - only log if explicitly told to
-            if ((user.isSystem() && SystemPrivilege.INSTANCE.predicate().test(action)) || ShieldInternalUserHolder.isShieldInternalUser(user)) {
+            if ((SystemUser.is(user) && SystemPrivilege.INSTANCE.predicate().test(action)) || XPackUser.is(user)) {
                 if (events.contains(SYSTEM_ACCESS_GRANTED)) {
                     try {
                         enqueue(message("access_granted", action, user, indices(message), message), "access_granted");
@@ -518,7 +516,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     }
 
     private boolean principalIsAuditor(String principal) {
-        return (principal.equals(auditUser.user().principal()));
+        return principal.equals(XPackUser.INSTANCE.principal());
     }
 
     private Message message(String type, @Nullable String action, @Nullable User user,
@@ -678,7 +676,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     private void initializeClient() {
         if (indexToRemoteCluster == false) {
             // in the absence of client settings for remote indexing, fall back to the client that was passed in.
-            this.client = new ClientWithUser(clientProvider.get(), authenticationService, auditUser.user());
+            this.client = clientProvider.get();
         } else {
             Settings clientSettings = settings.getByPrefix("shield.audit.index.client.");
             String[] hosts = clientSettings.getAsArray("hosts");
@@ -763,9 +761,6 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
 
             assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
 
-            if (!indexToRemoteCluster) {
-                authenticationService.attachUserHeaderIfMissing(auditUser.user());
-            }
             PutIndexTemplateResponse response = client.admin().indices().putTemplate(request).actionGet();
             if (!response.isAcknowledged()) {
                 throw new IllegalStateException("failed to put index template for audit logging");

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java

@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.audit.index;
-
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
-import org.elasticsearch.action.bulk.BulkAction;
-import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.authz.permission.Role;
-import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.shield.authz.privilege.IndexPrivilege;
-
-/**
- *
- */
-public class IndexAuditUserHolder {
-
-    private static final String NAME = "__indexing_audit_user";
-    private static final String[] ROLE_NAMES = new String[] { "__indexing_audit_role" };
-    public static final Role ROLE = Role.builder(ROLE_NAMES[0])
-        .cluster(ClusterPrivilege.action(PutIndexTemplateAction.NAME))
-        .add(IndexPrivilege.CREATE_INDEX, IndexAuditTrail.INDEX_NAME_PREFIX + "*")
-        .add(IndexPrivilege.INDEX, IndexAuditTrail.INDEX_NAME_PREFIX + "*")
-        .add(IndexPrivilege.action(IndicesExistsAction.NAME), IndexAuditTrail.INDEX_NAME_PREFIX + "*")
-        .add(IndexPrivilege.action(BulkAction.NAME), IndexAuditTrail.INDEX_NAME_PREFIX + "*")
-        .add(IndexPrivilege.action(PutMappingAction.NAME), IndexAuditTrail.INDEX_NAME_PREFIX + "*")
-        .build();
-
-    private final User user;
-
-    public IndexAuditUserHolder() {
-        this.user = new User(NAME, ROLE_NAMES);
-    }
-
-    public User user() {
-        return user;
-    }
-}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java

@@ -17,11 +17,11 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.shield.SystemUser;
 import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
+import org.elasticsearch.shield.XPackUser;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationToken;
-import org.elasticsearch.shield.authz.privilege.Privilege;
 import org.elasticsearch.shield.authz.privilege.SystemPrivilege;
 import org.elasticsearch.shield.rest.RemoteHostHeader;
 import org.elasticsearch.shield.transport.filter.ShieldIpFilterRule;
@@ -200,7 +200,7 @@ public class LoggingAuditTrail extends AbstractLifecycleComponent<LoggingAuditTr
         String indices = indicesString(message);
 
         // special treatment for internal system actions - only log on trace
-        if ((user.isSystem() && SystemPrivilege.INSTANCE.predicate().test(action)) || ShieldInternalUserHolder.isShieldInternalUser(user)) {
+        if ((SystemUser.is(user) && SystemPrivilege.INSTANCE.predicate().test(action)) || XPackUser.is(user)) {
             if (logger.isTraceEnabled()) {
                 if (indices != null) {
                     logger.trace("{}[transport] [access_granted]\t{}, {}, action=[{}], indices=[{}], request=[{}]", prefix, originAttributes(message, transport, threadContext), principal(user), action, indices, message.getClass().getSimpleName());

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java

@@ -12,11 +12,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
@@ -30,7 +26,6 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -44,27 +39,24 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.shield.InternalClient;
+import org.elasticsearch.shield.ShieldTemplateService;
 import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.action.admin.user.AddUserRequest;
+import org.elasticsearch.shield.action.realm.ClearRealmCacheRequest;
-import org.elasticsearch.shield.action.admin.user.DeleteUserRequest;
+import org.elasticsearch.shield.action.realm.ClearRealmCacheResponse;
-import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequest;
+import org.elasticsearch.shield.action.user.AddUserRequest;
-import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheResponse;
+import org.elasticsearch.shield.action.user.DeleteUserRequest;
-import org.elasticsearch.shield.admin.ShieldTemplateService;
-import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
 import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.client.ShieldClient;
-import org.elasticsearch.shield.support.ClientWithUser;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -96,9 +88,7 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat
     private final Hasher hasher = Hasher.BCRYPT;
     private final List<ChangeListener> listeners = new CopyOnWriteArrayList<>();
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
-    private final Provider<Client> clientProvider;
+    private final Provider<InternalClient> clientProvider;
-    private final Provider<AuthenticationService> authProvider;
-    private final ShieldInternalUserHolder adminUser;
     private final ThreadPool threadPool;
 
     private ScheduledFuture<?> versionChecker;
@@ -110,13 +100,10 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat
     private volatile boolean shieldIndexExists = false;
 
     @Inject
-    public ESNativeUsersStore(Settings settings, Provider<Client> clientProvider,
+    public ESNativeUsersStore(Settings settings, Provider<InternalClient> clientProvider,
-                              ShieldInternalUserHolder userHolder,
                               Provider<AuthenticationService> authProvider, ThreadPool threadPool) {
         super(settings);
         this.clientProvider = clientProvider;
-        this.authProvider = authProvider;
-        this.adminUser = userHolder;
         this.threadPool = threadPool;
     }
 
@@ -408,8 +395,7 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat
     public void start() {
         try {
             if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
-                this.authService = authProvider.get();
+                this.client = clientProvider.get();
-                this.client = new ClientWithUser(clientProvider.get(), authService, adminUser.user());
                 this.scrollSize = settings.getAsInt("shield.authc.native.scroll.size", 1000);
                 this.scrollKeepAlive = settings.getAsTime("shield.authc.native.scroll.keep_alive", TimeValue.timeValueSeconds(10L));
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/tool/ESUsersTool.java

@@ -50,7 +50,7 @@ public class ESUsersTool extends CliTool {
             .cmds(Useradd.CMD, Userdel.CMD, Passwd.CMD, Roles.CMD, ListUsersAndRoles.CMD)
             .build();
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws Exception {
         ExitStatus exitStatus = new ESUsersTool().execute(args);
         exit(exitStatus.status());
     }
@@ -160,7 +160,7 @@ public class ESUsersTool extends CliTool {
             Path file = FileUserPasswdStore.resolveFile(esusersSettings, env);
             Map<String, char[]> users = new HashMap<>(FileUserPasswdStore.parseFile(file, null));
             if (users.containsKey(username)) {
-                terminal.println("User [%s] already exists", username);
+                terminal.println(String.format("User [%s] already exists", username));
                 return ExitStatus.CODE_ERROR;
             }
             Hasher hasher = Hasher.BCRYPT;
@@ -228,7 +228,7 @@ public class ESUsersTool extends CliTool {
             Path file = FileUserPasswdStore.resolveFile(esusersSettings, env);
             Map<String, char[]> users = new HashMap<>(FileUserPasswdStore.parseFile(file, null));
             if (!users.containsKey(username)) {
-                terminal.println("User [%s] doesn't exist", username);
+                terminal.println(String.format("User [%s] doesn't exist", username));
                 return ExitStatus.NO_USER;
             }
 
@@ -308,7 +308,7 @@ public class ESUsersTool extends CliTool {
             Path file = FileUserPasswdStore.resolveFile(esusersSettings, env);
             Map<String, char[]> users = new HashMap<>(FileUserPasswdStore.parseFile(file, null));
             if (!users.containsKey(username)) {
-                terminal.println("User [%s] doesn't exist", username);
+                terminal.println(String.format("User [%s] doesn't exist", username));
                 return ExitStatus.NO_USER;
             }
             Hasher hasher = Hasher.BCRYPT;
@@ -378,7 +378,7 @@ public class ESUsersTool extends CliTool {
             String[] allRoles = ArrayUtils.concat(addRoles, removeRoles, String.class);
             for (String role : allRoles) {
                 if (!ROLE_PATTERN.matcher(role).matches()) {
-                    terminal.println("Role name [%s] is not valid. Please use lowercase and numbers only", role);
+                    terminal.println(String.format("Role name [%s] is not valid. Please use lowercase and numbers only", role));
                     return ExitStatus.DATA_ERROR;
                 }
             }
@@ -388,7 +388,7 @@ public class ESUsersTool extends CliTool {
             Path path = FileUserPasswdStore.resolveFile(esusersSettings, env);
             Map<String, char[]> usersMap = FileUserPasswdStore.parseFile(path, null);
             if (!usersMap.containsKey(username)) {
-                terminal.println("User [%s] doesn't exist", username);
+                terminal.println(String.format("User [%s] doesn't exist", username));
                 return ExitStatus.NO_USER;
             }
 
@@ -451,7 +451,7 @@ public class ESUsersTool extends CliTool {
 
             if (username != null) {
                 if (!users.contains(username)) {
-                    terminal.println("User [%s] doesn't exist", username);
+                    terminal.println(String.format("User [%s] doesn't exist", username));
                     return ExitStatus.NO_USER;
                 }
 
@@ -459,15 +459,15 @@ public class ESUsersTool extends CliTool {
                     String[] roles = userRoles.get(username);
                     Set<String> unknownRoles = Sets.difference(Sets.newHashSet(roles), knownRoles);
                     String[] markedRoles = markUnknownRoles(roles, unknownRoles);
-                    terminal.println("%-15s: %s", username, Arrays.stream(markedRoles).map(s -> s == null ? "-" : s).collect(Collectors.joining(",")));
+                    terminal.println(String.format("%-15s: %s", username, Arrays.stream(markedRoles).map(s -> s == null ? "-" : s).collect(Collectors.joining(","))));
                     if (!unknownRoles.isEmpty()) {
                         // at least one role is marked... so printing the legend
                         Path rolesFile = FileRolesStore.resolveFile(esusersSettings, env).toAbsolutePath();
                         terminal.println();
-                        terminal.println(" [*]   An unknown role. Please check [%s] to see available roles", rolesFile.toAbsolutePath());
+                        terminal.println(String.format(" [*]   An unknown role. Please check [%s] to see available roles", rolesFile.toAbsolutePath()));
                     }
                 } else {
-                    terminal.println("%-15s: -", username);
+                    terminal.println(String.format("%-15s: -", username));
                 }
             } else {
                 boolean unknownRolesFound = false;
@@ -476,7 +476,7 @@ public class ESUsersTool extends CliTool {
                     String[] roles = entry.getValue();
                     Set<String> unknownRoles = Sets.difference(Sets.newHashSet(roles), knownRoles);
                     String[] markedRoles = markUnknownRoles(roles, unknownRoles);
-                    terminal.println("%-15s: %s", entry.getKey(), String.join(",", markedRoles));
+                    terminal.println(String.format("%-15s: %s", entry.getKey(), String.join(",", markedRoles)));
                     unknownRolesFound = unknownRolesFound || !unknownRoles.isEmpty();
                     usersExist = true;
                 }
@@ -484,7 +484,7 @@ public class ESUsersTool extends CliTool {
                 Set<String> usersWithoutRoles = Sets.newHashSet(users);
                 usersWithoutRoles.removeAll(userRoles.keySet());
                 for (String user : usersWithoutRoles) {
-                    terminal.println("%-15s: -", user);
+                    terminal.println(String.format("%-15s: -", user));
                     usersExist = true;
                 }
 
@@ -497,7 +497,7 @@ public class ESUsersTool extends CliTool {
                     // at least one role is marked... so printing the legend
                     Path rolesFile = FileRolesStore.resolveFile(esusersSettings, env).toAbsolutePath();
                     terminal.println();
-                    terminal.println(" [*]   An unknown role. Please check [%s] to see available roles", rolesFile.toAbsolutePath());
+                    terminal.println(String.format(" [*]   An unknown role. Please check [%s] to see available roles", rolesFile.toAbsolutePath()));
                 }
             }
 
@@ -511,7 +511,7 @@ public class ESUsersTool extends CliTool {
             return FileRolesStore.parseFileForRoleNames(rolesFile, null);
         } catch (Throwable t) {
             // if for some reason, parsing fails (malformatted perhaps) we just warn
-            terminal.println("Warning:  Could not parse [%s] for roles verification. Please revise and fix it. Nonetheless, the user will still be associated with all specified roles", rolesFile.toAbsolutePath());
+            terminal.println(String.format("Warning:  Could not parse [%s] for roles verification. Please revise and fix it. Nonetheless, the user will still be associated with all specified roles", rolesFile.toAbsolutePath()));
         }
         return null;
     }
@@ -536,9 +536,9 @@ public class ESUsersTool extends CliTool {
         Set<String> unknownRoles = Sets.difference(Sets.newHashSet(roles), knownRoles);
         if (!unknownRoles.isEmpty()) {
             Path rolesFile = FileRolesStore.resolveFile(settings, env);
-            terminal.println("Warning: The following roles [%s] are unknown. Make sure to add them to the [%s] file. " +
+            terminal.println(String.format("Warning: The following roles [%s] are unknown. Make sure to add them to the [%s] file. " +
                     "Nonetheless the user will still be associated with all specified roles",
-                    Strings.collectionToCommaDelimitedString(unknownRoles), rolesFile.toAbsolutePath());
+                Strings.collectionToCommaDelimitedString(unknownRoles), rolesFile.toAbsolutePath()));
         }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java

@@ -5,42 +5,26 @@
  */
 package org.elasticsearch.shield.authz;
 
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
 import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
-import org.elasticsearch.shield.authz.permission.Role;
 import org.elasticsearch.shield.authz.store.CompositeRolesStore;
 import org.elasticsearch.shield.authz.store.FileRolesStore;
 import org.elasticsearch.shield.authz.store.RolesStore;
 import org.elasticsearch.shield.support.AbstractShieldModule;
 
-import java.util.HashSet;
-import java.util.Set;
-
 /**
  * Module used to bind various classes necessary for authorization
  */
 public class AuthorizationModule extends AbstractShieldModule.Node {
 
-    private final Set<Role> reservedRoles = new HashSet<>();
-
     public AuthorizationModule(Settings settings) {
         super(settings);
     }
 
-    public void registerReservedRole(Role role) {
-        reservedRoles.add(role);
-    }
-
     @Override
     protected void configureNode() {
 
-        Multibinder<Role> reservedRolesBinder = Multibinder.newSetBinder(binder(), Role.class);
-        for (Role reservedRole : reservedRoles) {
-            reservedRolesBinder.addBinding().toInstance(reservedRole);
-        }
-
         // First the file and native roles stores must be bound...
         bind(FileRolesStore.class).asEagerSingleton();
         bind(ESNativeRolesStore.class).asEagerSingleton();

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationUtils.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz;
+
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.SystemUser;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.InternalAuthenticationService;
+import org.elasticsearch.shield.support.AutomatonPredicate;
+import org.elasticsearch.shield.support.Automatons;
+
+import java.util.function.Predicate;
+
+/**
+ *
+ */
+public final class AuthorizationUtils {
+
+    private static final Predicate<String> INTERNAL_PREDICATE = new AutomatonPredicate(Automatons.patterns("internal:*"));
+
+    private AuthorizationUtils() {}
+
+    /**
+     * This method is used to determine if a request should be executed as the system user, even if the request already
+     * has a user associated with it.
+     *
+     * In order for the system user to be used, one of the following conditions must be true:
+     *
+     * <ul>
+     *     <li>the action is an internal action and no user is associated with the request</li>
+     *     <li>the action is an internal action and the system user is already associated with the request</li>
+     *     <li>the action is an internal action and the thread context contains a non-internal action as the originating action</li>
+     * </ul>
+     *
+     * @param threadContext the {@link ThreadContext} that contains the headers and context associated with the request
+     * @param action the action name that is being executed
+     * @return true if the system user should be used to execute a request
+     */
+    public static boolean shouldReplaceUserWithSystem(ThreadContext threadContext, String action) {
+        if (isInternalAction(action) == false) {
+            return false;
+        }
+
+        User user = threadContext.getTransient(InternalAuthenticationService.USER_KEY);
+        if (user == null || SystemUser.is(user)) {
+            return true;
+        }
+
+        // we have a internal action being executed by a user that is not the system user, lets verify that there is a
+        // originating action that is not a internal action
+        final String originatingAction = threadContext.getTransient(InternalAuthorizationService.ORIGINATING_ACTION_KEY);
+        if (originatingAction != null && isInternalAction(originatingAction) == false) {
+            return true;
+        }
+
+        // either there was no originating action or it was a internal action, we should not replace under these circumstances
+        return false;
+    }
+
+    public static boolean isInternalAction(String action) {
+        return INTERNAL_PREDICATE.test(action);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java

@@ -22,7 +22,9 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.search.action.SearchServiceTransportAction;
+import org.elasticsearch.shield.SystemUser;
 import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.XPackUser;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AnonymousService;
 import org.elasticsearch.shield.authc.AuthenticationFailureHandler;
@@ -54,6 +56,7 @@ import static org.elasticsearch.shield.support.Exceptions.authorizationError;
 public class InternalAuthorizationService extends AbstractComponent implements AuthorizationService {
 
     public static final String INDICES_PERMISSIONS_KEY = "_indices_permissions";
+    static final String ORIGINATING_ACTION_KEY = "_originating_action_name";
 
     private final ClusterService clusterService;
     private final RolesStore rolesStore;
@@ -81,6 +84,10 @@ public class InternalAuthorizationService extends AbstractComponent implements A
 
     @Override
     public List<String> authorizedIndicesAndAliases(User user, String action) {
+        Predicate<String> predicate;
+        if (XPackUser.is(user)) {
+            predicate = XPackUser.ROLE.indices().allowedIndicesMatcher(action);
+        } else {
             String[] rolesNames = user.roles();
             if (rolesNames.length == 0) {
                 return Collections.emptyList();
@@ -92,9 +99,10 @@ public class InternalAuthorizationService extends AbstractComponent implements A
                     predicates.add(role.indices().allowedIndicesMatcher(action));
                 }
             }
+            predicate = predicates.stream().reduce(s -> false, (p1, p2) -> p1.or(p2));
+        }
 
         List<String> indicesAndAliases = new ArrayList<>();
-        Predicate<String> predicate = predicates.stream().reduce(s -> false, (p1, p2) -> p1.or(p2));
         MetaData metaData = clusterService.state().metaData();
         // TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
         for (Map.Entry<String, AliasOrIndex> entry : metaData.getAliasAndIndexLookup().entrySet()) {
@@ -103,14 +111,18 @@ public class InternalAuthorizationService extends AbstractComponent implements A
                 indicesAndAliases.add(aliasOrIndex);
             }
         }
+
         return Collections.unmodifiableList(indicesAndAliases);
     }
 
     @Override
     public void authorize(User user, String action, TransportRequest request) throws ElasticsearchSecurityException {
+        // prior to doing any authorization lets set the originating action in the context only
+        setOriginatingAction(action);
+
         // first we need to check if the user is the system. If it is, we'll just authorize the system access
-        if (user.isSystem()) {
+        if (SystemUser.is(user)) {
-            if (SystemRole.INSTANCE.check(action)) {
+            if (SystemUser.isAuthorized(action)) {
                 setIndicesAccessControl(IndicesAccessControl.ALLOW_ALL);
                 grant(user, action, request);
                 return;
@@ -118,7 +130,8 @@ public class InternalAuthorizationService extends AbstractComponent implements A
             throw denial(user, action, request);
         }
 
-        GlobalPermission permission = permission(user.roles());
+        GlobalPermission permission = XPackUser.is(user) ? XPackUser.ROLE : permission(user.roles());
+
         final boolean isRunAs = user.runAs() != null;
         // permission can be null as it might be that the user's role
         // is unknown
@@ -226,6 +239,13 @@ public class InternalAuthorizationService extends AbstractComponent implements A
         }
     }
 
+    private void setOriginatingAction(String action) {
+        String originatingAction = threadContext.getTransient(ORIGINATING_ACTION_KEY);
+        if (originatingAction == null) {
+            threadContext.putTransient(ORIGINATING_ACTION_KEY, action);
+        }
+    }
+
     private GlobalPermission permission(String[] roleNames) {
         if (roleNames.length == 0) {
             return GlobalPermission.NONE;
@@ -240,7 +260,7 @@ public class InternalAuthorizationService extends AbstractComponent implements A
 
         GlobalPermission.Compound.Builder roles = GlobalPermission.Compound.builder();
         for (String roleName : roleNames) {
-            GlobalPermission role = rolesStore.role(roleName);
+            Role role = rolesStore.role(roleName);
             if (role != null) {
                 roles.add(role);
             }