Add test for global checkpoint sync with security

After a write operation on an index, a post-operation global checkpoint
sync fires. Previously, this action fired on the same user as executed
the write action. If the user did not have priviledges for this action,
the global checkpoint sync would fail. With an upstream change in core,
this action now fires as the system user. This commit adds a test that
create a user that has minimal write permissions on an index, but none
that would imply it could execute the global checkpoint sync. This then
serves as a test that the upstream change to fire the global checkpoint
sync as the system user is correct. This test must run as a mulit-node
test so that a replica is a assigned so that the global checkpoint sync
fires in the first place. This test does indeed fail without the
upstream change, and passes with it.

Relates elastic/x-pack-elasticsearch#2744

Original commit: elastic/x-pack-elasticsearch@bf7e771756

qa/multi-node/build.gradle

@@ -0,0 +1,38 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+dependencies {
+  testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime')
+  testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts')
+}
+
+task multiNodeTest(type: RestIntegTestTask) {
+  mustRunAfter(precommit)
+}
+
+multiNodeTestCluster {
+  distribution = 'zip'
+  numNodes = 2
+  clusterName = 'multi-node'
+  plugin ':x-pack-elasticsearch:plugin'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  extraConfigFile 'x-pack/roles.yml', 'roles.yml'
+  setupCommand 'setup-test-user', 'bin/x-pack/users', 'useradd', 'test-user', '-p', 'x-pack-test-password', '-r', 'test'
+  setupCommand 'setup-super-user', 'bin/x-pack/users', 'useradd', 'super-user', '-p', 'x-pack-super-password', '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'super-user',
+            password: 'x-pack-super-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
+}
+
+test.enabled = false // no unit tests for multi-node, only the rest integration test
+check.dependsOn(multiNodeTest)

qa/multi-node/roles.yml

@@ -0,0 +1,14 @@
+test:
+  # this privileges must be kept minimal or the user could inadvertently be
+  # granted permissions that would enable the global checkpoint sync to run
+  # as this user leading to tests that should fail if the global checkpoint
+  # sync does not run as the system user to pass for when they should not
+  cluster:
+    - monitor
+  indices:
+    - names: [ 'test-index' ]
+      privileges:
+        - create_index
+        - monitor
+        - write
+

qa/multi-node/src/test/java/org/elasticsearch/multi_node/MultiNodeIT.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.multi_node;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase;
+
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public class MultiNodeIT extends SecurityClusterClientYamlTestCase {
+
+    public MultiNodeIT(@Name("yaml") final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        return getClientSettings("test-user", "x-pack-test-password");
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        return getClientSettings("super-user", "x-pack-super-password");
+    }
+
+    private Settings getClientSettings(final String username, final String password) {
+        final String token = basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+}

qa/multi-node/src/test/resources/rest-api-spec/test/global_checkpoint_sync/10_basic.yml

@@ -0,0 +1,31 @@
+---
+"Global checkpoint sync action runs as a privileged user":
+
+ - do:
+     indices.create:
+       index: test-index
+       body:
+         settings:
+           index:
+             number_of_shards: 1
+             number_of_replicas: 1
+
+ - do:
+     cluster.health:
+       wait_for_status: green
+
+ - do:
+     index:
+       index: test-index
+       type: test-type
+       id: 1
+       body: { foo: bar }
+
+ - do:
+     indices.stats:
+       index: test-index
+       level: shards
+       filter_path: "**.seq_no"
+
+ - match: { indices.test-index.shards.0.0.seq_no.global_checkpoint: 0 }
+ - match: { indices.test-index.shards.0.1.seq_no.global_checkpoint: 0 }