Add role for enrich processor (#42677)

This commit adds the manage_enrich privilege, which grants access to all
of the enrich processor lifecycle actions. In addition this commit also
creates a role which grants access to the generated indices.

Relates #41939

Co-authored-by: Martijn van Groningen <martijn.v.groningen@gmail.com>

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java

@@ -318,9 +318,11 @@ public final class Role {
         public static final String READ_CCR = "read_ccr";
         public static final String MANAGE_ILM = "manage_ilm";
         public static final String READ_ILM = "read_ilm";
+        public static final String MANAGE_ENRICH = "manage_enrich";
         public static final String[] ALL_ARRAY = new String[] { NONE, ALL, MONITOR, MONITOR_ML, MONITOR_WATCHER, MONITOR_ROLLUP, MANAGE,
                 MANAGE_ML, MANAGE_WATCHER, MANAGE_ROLLUP, MANAGE_INDEX_TEMPLATES, MANAGE_INGEST_PIPELINES, TRANSPORT_CLIENT,
-            MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM};
+                MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM,
+                MANAGE_ENRICH };
     }
 
     /**

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -670,7 +670,7 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
             List<Role> roles = response.getRoles();
             assertNotNull(response);
             // 27 system roles plus the three we created
-            assertThat(roles.size(), equalTo(30));
+            assertThat(roles.size(), equalTo(31));
         }
 
         {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilege.java

@@ -60,6 +60,7 @@ public final class ClusterPrivilege extends Privilege {
     private static final Automaton READ_CCR_AUTOMATON = patterns(ClusterStateAction.NAME, HasPrivilegesAction.NAME);
     private static final Automaton MANAGE_ILM_AUTOMATON = patterns("cluster:admin/ilm/*");
     private static final Automaton READ_ILM_AUTOMATON = patterns(GetLifecycleAction.NAME, GetStatusAction.NAME);
+    private static final Automaton MANAGE_ENRICH_AUTOMATON = patterns("cluster:admin/xpack/enrich/*");
 
     public static final ClusterPrivilege NONE =                  new ClusterPrivilege("none",                Automatons.EMPTY);
     public static final ClusterPrivilege ALL =                   new ClusterPrivilege("all",                 ALL_CLUSTER_AUTOMATON);
@@ -90,6 +91,7 @@ public final class ClusterPrivilege extends Privilege {
     public static final ClusterPrivilege CREATE_SNAPSHOT =       new ClusterPrivilege("create_snapshot", CREATE_SNAPSHOT_AUTOMATON);
     public static final ClusterPrivilege MANAGE_ILM =            new ClusterPrivilege("manage_ilm", MANAGE_ILM_AUTOMATON);
     public static final ClusterPrivilege READ_ILM =              new ClusterPrivilege("read_ilm", READ_ILM_AUTOMATON);
+    public static final ClusterPrivilege MANAGE_ENRICH =         new ClusterPrivilege("manage_enrich", MANAGE_ENRICH_AUTOMATON);
 
     public static final Predicate<String> ACTION_MATCHER = ClusterPrivilege.ALL.predicate();
 
@@ -119,6 +121,7 @@ public final class ClusterPrivilege extends Privilege {
             .put("create_snapshot", CREATE_SNAPSHOT)
             .put("manage_ilm", MANAGE_ILM)
             .put("read_ilm", READ_ILM)
+            .put("manage_enrich", MANAGE_ENRICH)
             .immutableMap();
 
     private static final ConcurrentHashMap<Set<String>, ClusterPrivilege> CACHE = new ConcurrentHashMap<>();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -233,6 +233,11 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                                 .privileges("view_index_metadata")
                                 .allowRestrictedIndices(true)
                                 .build() }, null, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
+                .put("enrich_user", new RoleDescriptor("enrich_user", new String[]{ "manage_enrich", "manage_ingest_pipelines", "monitor" },
+                        new RoleDescriptor.IndicesPrivileges[]{  RoleDescriptor.IndicesPrivileges.builder()
+                            .indices(".enrich-*")
+                            .privileges("manage", "read", "write")
+                            .build() }, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .immutableMap();
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java

@@ -8,6 +8,11 @@ package org.elasticsearch.xpack.core.security.authz.privilege;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction;
+import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
+import org.elasticsearch.xpack.core.enrich.action.GetEnrichPolicyAction;
+import org.elasticsearch.xpack.core.enrich.action.ListEnrichPolicyAction;
+import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
@@ -151,6 +156,17 @@ public class PrivilegeTests extends ESTestCase {
         assertThat(predicate.test("cluster:admin/xpack/whatever"), is(false));
     }
 
+    public void testManageEnrichPrivilege() {
+        Predicate<String> predicate = ClusterPrivilege.MANAGE_ENRICH.predicate();
+        assertThat(predicate.test(DeleteEnrichPolicyAction.NAME), is(true));
+        assertThat(predicate.test(ExecuteEnrichPolicyAction.NAME), is(true));
+        assertThat(predicate.test(GetEnrichPolicyAction.NAME), is(true));
+        assertThat(predicate.test(ListEnrichPolicyAction.NAME), is(true));
+        assertThat(predicate.test(PutEnrichPolicyAction.NAME), is(true));
+        assertThat(predicate.test("cluster:admin/xpack/enrich/brand_new_api"), is(true));
+        assertThat(predicate.test("cluster:admin/xpack/whatever"), is(false));
+    }
+
     public void testIlmPrivileges() {
         {
             Predicate<String> predicate = ClusterPrivilege.MANAGE_ILM.predicate();

x-pack/plugin/enrich/qa/common/build.gradle

@@ -0,0 +1,6 @@
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+    compile project(':test:framework')
+}

x-pack/plugin/enrich/qa/common/src/main/java/org/elasticsearch/test/enrich/CommonEnrichRestTestCase.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.test.enrich;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.After;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class CommonEnrichRestTestCase extends ESRestTestCase {
+
+    @After
+    private void deletePolicies() throws Exception {
+        Map<String, Object> responseMap = toMap(adminClient().performRequest(new Request("GET", "/_enrich/policy")));
+        @SuppressWarnings("unchecked")
+        List<Map<?,?>> policies = (List<Map<?,?>>) responseMap.get("policies");
+
+        for (Map<?, ?> entry: policies) {
+            client().performRequest(new Request("DELETE", "/_enrich/policy/" + entry.get("name")));
+        }
+    }
+
+    public void testBasicFlow() throws Exception {
+        // Create the policy:
+        Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy");
+        putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-source-index\"], \"enrich_key\": \"host\", " +
+            "\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"]}");
+        assertOK(client().performRequest(putPolicyRequest));
+
+        // Add entry to source index and then refresh:
+        Request indexRequest = new Request("PUT", "/my-source-index/_doc/elastic.co");
+        indexRequest.setJsonEntity("{\"host\": \"elastic.co\",\"globalRank\": 25,\"tldRank\": 7,\"tld\": \"co\"}");
+        assertOK(client().performRequest(indexRequest));
+        Request refreshRequest = new Request("POST", "/my-source-index/_refresh");
+        assertOK(client().performRequest(refreshRequest));
+
+        // Execute the policy:
+        Request executePolicyRequest = new Request("POST", "/_enrich/policy/my_policy/_execute");
+        assertOK(client().performRequest(executePolicyRequest));
+
+        // Create pipeline
+        Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/my_pipeline");
+        putPipelineRequest.setJsonEntity("{\"processors\":[" +
+            "{\"enrich\":{\"policy_name\":\"my_policy\",\"enrich_key\":\"host\",\"enrich_values\":[" +
+            "{\"source\":\"globalRank\",\"target\":\"global_rank\"}," +
+            "{\"source\":\"tldRank\",\"target\":\"tld_rank\"}" +
+            "]}}" +
+            "]}");
+        assertOK(client().performRequest(putPipelineRequest));
+
+        // Index document using pipeline with enrich processor:
+        indexRequest = new Request("PUT", "/my-index/_doc/1");
+        indexRequest.addParameter("pipeline", "my_pipeline");
+        indexRequest.setJsonEntity("{\"host\": \"elastic.co\"}");
+        assertOK(client().performRequest(indexRequest));
+
+        // Check if document has been enriched
+        Request getRequest = new Request("GET", "/my-index/_doc/1");
+        Map<String, Object> response = toMap(client().performRequest(getRequest));
+        Map<?, ?> _source = (Map<?, ?>) response.get("_source");
+        assertThat(_source.size(), equalTo(3));
+        assertThat(_source.get("host"), equalTo("elastic.co"));
+        assertThat(_source.get("global_rank"), equalTo(25));
+        assertThat(_source.get("tld_rank"), equalTo(7));
+    }
+
+    private static Map<String, Object> toMap(Response response) throws IOException {
+        return toMap(EntityUtils.toString(response.getEntity()));
+    }
+
+    private static Map<String, Object> toMap(String response) {
+        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
+    }
+}

x-pack/plugin/enrich/qa/rest-with-security/build.gradle

@@ -0,0 +1,38 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+dependencies {
+    testCompile project(path: xpackModule('enrich'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('enrich:qa:common'), configuration: 'runtime')}
+
+task restTest(type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+}
+
+restTestCluster {
+    distribution 'default'
+    setting 'xpack.license.self_generated.type', 'basic'
+    setting 'xpack.security.enabled', 'true'
+    extraConfigFile 'roles.yml', 'roles.yml'
+    setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+    setupCommand 'setupEnrichUser',
+            'bin/elasticsearch-users', 'useradd', "test_enrich", '-p', 'x-pack-test-password', '-r', "enrich_user,integ_test_role"
+    setupCommand 'setupEnrichUserNoPrivs',
+            'bin/elasticsearch-users', 'useradd', "test_enrich_no_privs", '-p', 'x-pack-test-password', '-r', "enrich_no_privs"
+    waitCondition = { node, ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+                dest: tmpFile.toString(),
+                username: 'test_admin',
+                password: 'x-pack-test-password',
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
+}
+
+check.dependsOn restTest
+test.enabled = false

x-pack/plugin/enrich/qa/rest-with-security/roles.yml

@@ -0,0 +1,18 @@
+integ_test_role:
+  indices:
+    - names: [ 'my-index', 'my-source-index' ]
+      privileges:
+        - manage
+        - read
+        - write
+
+enrich_no_privs:
+  cluster:
+    - manage_ingest_pipelines
+    - monitor
+  indices:
+    - names: [ '.enrich-my_policy*', 'my-index', 'my-source-index' ]
+      privileges:
+        - manage
+        - read
+        - write

x-pack/plugin/enrich/qa/rest-with-security/src/test/java/org/elasticsearch/xpack/enrich/EnrichSecurityFailureIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.enrich;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public class EnrichSecurityFailureIT extends ESRestTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test_enrich_no_privs", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    public void testFailure() {
+        Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy");
+        putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-source-index\"], \"enrich_key\": \"host\", " +
+            "\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"]}");
+        ResponseException exc = expectThrows(ResponseException.class, () -> assertOK(client().performRequest(putPolicyRequest)));
+        assertTrue(exc.getMessage().contains("action [cluster:admin/xpack/enrich/put] is unauthorized for user [test_enrich_no_privs]"));
+    }
+}

x-pack/plugin/enrich/qa/rest-with-security/src/test/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.enrich;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public class EnrichSecurityIT extends CommonEnrichRestTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test_enrich", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+}

x-pack/plugin/enrich/qa/rest/build.gradle

@@ -4,6 +4,7 @@ apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
     testCompile project(path: xpackModule('enrich'), configuration: 'runtime')
+    testCompile project(path: xpackModule('enrich:qa:common'), configuration: 'runtime')
 }
 
 task restTest(type: RestIntegTestTask) {

x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java

@@ -5,83 +5,7 @@
  */
 package org.elasticsearch.xpack.enrich;
 
-import org.apache.http.util.EntityUtils;
+import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.After;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class EnrichIT extends ESRestTestCase {
-
-    @After
-    private void deletePolicies() throws Exception {
-        Map<String, Object> responseMap = toMap(client().performRequest(new Request("GET", "/_enrich/policy")));
-        @SuppressWarnings("unchecked")
-        List<Map<?,?>> policies = (List<Map<?,?>>) responseMap.get("policies");
-
-        for (Map<?, ?> entry: policies) {
-            client().performRequest(new Request("DELETE", "/_enrich/policy/" + entry.get("name")));
-        }
-    }
-
-    public void testBasicFlow() throws Exception {
-        // Create the policy:
-        Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy");
-        putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-source-index\"], \"enrich_key\": \"host\", " +
-            "\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"]}");
-        assertOK(client().performRequest(putPolicyRequest));
-
-        // Add entry to source index and then refresh:
-        Request indexRequest = new Request("PUT", "/my-source-index/_doc/elastic.co");
-        indexRequest.setJsonEntity("{\"host\": \"elastic.co\",\"globalRank\": 25,\"tldRank\": 7,\"tld\": \"co\"}");
-        assertOK(client().performRequest(indexRequest));
-        Request refreshRequest = new Request("POST", "/my-source-index/_refresh");
-        assertOK(client().performRequest(refreshRequest));
-
-        // Execute the policy:
-        Request executePolicyRequest = new Request("POST", "/_enrich/policy/my_policy/_execute");
-        assertOK(client().performRequest(executePolicyRequest));
-
-        // Create pipeline
-        Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/my_pipeline");
-        putPipelineRequest.setJsonEntity("{\"processors\":[" +
-            "{\"enrich\":{\"policy_name\":\"my_policy\",\"enrich_key\":\"host\",\"enrich_values\":[" +
-                "{\"source\":\"globalRank\",\"target\":\"global_rank\"}," +
-                "{\"source\":\"tldRank\",\"target\":\"tld_rank\"}" +
-                "]}}" +
-            "]}");
-        assertOK(client().performRequest(putPipelineRequest));
-
-        // Index document using pipeline with enrich processor:
-        indexRequest = new Request("PUT", "/my-index/_doc/1");
-        indexRequest.addParameter("pipeline", "my_pipeline");
-        indexRequest.setJsonEntity("{\"host\": \"elastic.co\"}");
-        assertOK(client().performRequest(indexRequest));
-
-        // Check if document has been enriched
-        Request getRequest = new Request("GET", "/my-index/_doc/1");
-        Map<String, Object> response = toMap(client().performRequest(getRequest));
-        Map<?, ?> _source = (Map<?, ?>) response.get("_source");
-        assertThat(_source.size(), equalTo(3));
-        assertThat(_source.get("host"), equalTo("elastic.co"));
-        assertThat(_source.get("global_rank"), equalTo(25));
-        assertThat(_source.get("tld_rank"), equalTo(7));
-    }
-
-    private static Map<String, Object> toMap(Response response) throws IOException {
-        return toMap(EntityUtils.toString(response.getEntity()));
-    }
-
-    private static Map<String, Object> toMap(String response) {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
-    }
 
+public class EnrichIT extends CommonEnrichRestTestCase {
 }