From 016b9d017cef6821ba631f4d0bc084e340cde9e0 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 8 Mar 2016 17:15:40 -0500 Subject: [PATCH] Add reindex tests to shield Original commit: elastic/x-pack-elasticsearch@fe00f317b3c366fe43a44417dab88ea070ff6dc0 --- .../qa/shield-reindex-tests/build.gradle | 35 ++ .../qa/shield-reindex-tests/roles.yml | 51 +++ .../java/org/elasticsearch/shield/RestIT.java | 45 +++ .../rest-api-spec/test/10_reindex.yaml | 317 ++++++++++++++++++ .../test/20_update_by_query.yaml | 213 ++++++++++++ 5 files changed, 661 insertions(+) create mode 100644 elasticsearch/qa/shield-reindex-tests/build.gradle create mode 100644 elasticsearch/qa/shield-reindex-tests/roles.yml create mode 100644 elasticsearch/qa/shield-reindex-tests/src/test/java/org/elasticsearch/shield/RestIT.java create mode 100644 elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/10_reindex.yaml create mode 100644 elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/20_update_by_query.yaml diff --git a/elasticsearch/qa/shield-reindex-tests/build.gradle b/elasticsearch/qa/shield-reindex-tests/build.gradle new file mode 100644 index 00000000000..4eec32ef935 --- /dev/null +++ b/elasticsearch/qa/shield-reindex-tests/build.gradle @@ -0,0 +1,35 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime') +} + +integTest { + cluster { + systemProperty 'es.script.inline', 'true' + plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') + extraConfigFile 'xpack/roles.yml', 'roles.yml' + [ + test_admin: 'admin', + powerful_user: 'admin', + minimal_user: 'minimal', + readonly_user: 'readonly', + dest_only_user: 'dest_only', + can_not_see_hidden_docs_user: 'can_not_see_hidden_docs', + can_not_see_hidden_fields_user: 'can_not_see_hidden_fields', + ].each { String user, String role -> + setupCommand 'setupUser#' + user, + 'bin/xpack/esusers', 'useradd', user, '-p', 'changeme', '-r', role + } + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://${node.httpUri()}", + dest: tmpFile.toString(), + username: 'test_admin', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} diff --git a/elasticsearch/qa/shield-reindex-tests/roles.yml b/elasticsearch/qa/shield-reindex-tests/roles.yml new file mode 100644 index 00000000000..cf188427cef --- /dev/null +++ b/elasticsearch/qa/shield-reindex-tests/roles.yml @@ -0,0 +1,51 @@ +admin: + cluster: all + indices: + '*': + privileges: all + run_as: '*' + +# Search and write on both source and destination indices. It should work if you could just search on the source and +# write to the destination but that isn't how shield works. +minimal: + indices: + source: + privileges: search, write, create_index, indices:admin/refresh + dest: + privileges: search, write, create_index, indices:admin/refresh + +# Read only operations on indices +readonly: + indices: + '*': + privileges: search + +# Write operations on destination index, none on source index +dest_only: + indices: + dest: + privileges: write + +# Search and write on both source and destination indices with document level security filtering out some docs. +can_not_see_hidden_docs: + indices: + source: + privileges: search, write, create_index, indices:admin/refresh + query: + bool: + must_not: + match: + hidden: true + dest: + privileges: search, write, create_index, indices:admin/refresh + +# Search and write on both source and destination indices with field level security. +can_not_see_hidden_fields: + indices: + source: + privileges: search, write, create_index, indices:admin/refresh + fields: + - foo + - bar + dest: + privileges: search, write, create_index, indices:admin/refresh diff --git a/elasticsearch/qa/shield-reindex-tests/src/test/java/org/elasticsearch/shield/RestIT.java b/elasticsearch/qa/shield-reindex-tests/src/test/java/org/elasticsearch/shield/RestIT.java new file mode 100644 index 00000000000..e49055a963d --- /dev/null +++ b/elasticsearch/qa/shield-reindex-tests/src/test/java/org/elasticsearch/shield/RestIT.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.shield; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +public class RestIT extends ESRestTestCase { + private static final String USER = "test_admin"; + private static final String PASS = "changeme"; + + public RestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } + + /** + * All tests run as a an administrative user but use es-shield-runas-user to become a less privileged user. + */ + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } +} + diff --git a/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/10_reindex.yaml b/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/10_reindex.yaml new file mode 100644 index 00000000000..d6a82dbc29d --- /dev/null +++ b/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/10_reindex.yaml @@ -0,0 +1,317 @@ +--- +"Reindex as same user works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + reindex: + body: + source: + index: source + dest: + index: dest + - match: {created: 1} + +--- +"Reindex with runas user works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: powerful_user} + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + - match: {created: 1} + + - do: + search: + index: dest + body: + query: + match: + text: test + - match: { hits.total: 1 } + + +--- +"Reindex with runas user with minimal privileges works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: minimal_user} + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + - match: {created: 1} + + - do: + search: + index: dest + body: + query: + match: + text: test + - match: { hits.total: 1 } + + +--- +"Reindex as readonly user is forbidden": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: readonly_user} + catch: forbidden + reindex: + body: + source: + index: source + dest: + index: dest + +--- +"Reindex as user that can't read from the source is forbidden": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: dest_only_user} + catch: forbidden + reindex: + body: + source: + index: source + dest: + index: dest + +--- +"Using a script to write to an index to which you don't have access is forbidden": + - do: + index: + index: source + type: tweet + id: 1 + body: { "user": "kimchy" } + - do: + index: + index: source + type: tweet + id: 2 + body: { "user": "another" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: dest_only_user} + catch: forbidden + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + script: + inline: if (ctx._source.user == "kimchy") {ctx._index = 'other_dest'} + + - do: + indices.refresh: {} + + # The index to which the user tried the unauthorized write didn't even get created + - do: + catch: missing + search: + index: other_dest + + # Even the authorized index won't have made it because it was in the same batch as the unauthorized one. + # If there had been lots of documents being copied then some might have made it into the authorized index. + - do: + catch: missing + search: + index: dest + +--- +"Reindex misses hidden docs": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + index: + index: source + type: foo + id: 2 + body: { "text": "test", "hidden": true } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_docs_user} + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + - match: {created: 1} + + # We copied just one doc, presumably the one without the hidden field + - do: + search: + index: dest + body: + query: + match: + text: test + - match: { hits.total: 1 } + + # We didn't copy the doc with the hidden field + - do: + search: + index: dest + body: + query: + match: + hidden: true + - match: { hits.total: 0 } + +--- +"Reindex misses hidden fields": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test", "foo": "z", "bar": "z" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_fields_user} + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + - match: {created: 1} + + - do: + search: + index: dest + body: + query: + match: + foo: z + - match: { hits.total: 1 } + + - do: + search: + index: dest + body: + query: + match: + bar: z + - match: { hits.total: 1 } + + - do: + search: + index: dest + body: + query: + match: + text: test + - match: { hits.total: 0 } + +--- +"Reindex to index with document level security is forbidden": + + - do: + index: + index: dest + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_docs_user} + reindex: + body: + source: + index: dest + dest: + index: source + +--- +"Reindex to index with field level security is forbidden": + + - do: + index: + index: dest + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_fields_user} + reindex: + body: + source: + index: dest + dest: + index: source diff --git a/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/20_update_by_query.yaml b/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/20_update_by_query.yaml new file mode 100644 index 00000000000..168dbb711af --- /dev/null +++ b/elasticsearch/qa/shield-reindex-tests/src/test/resources/rest-api-spec/test/20_update_by_query.yaml @@ -0,0 +1,213 @@ +--- +"Update-by-query as same user works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + update-by-query: + refresh: true + index: source + body: + script: + inline: ctx._source['hi'] = 'there' + - match: {updated: 1} + + - do: + search: + index: source + body: + query: + match: + hi: there + - match: { hits.total: 1 } + +--- +"Update-by-query with runas user works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: powerful_user} + update-by-query: + refresh: true + index: source + body: + script: + inline: ctx._source['hi'] = 'there' + - match: {updated: 1} + + - do: + search: + index: source + body: + query: + match: + hi: there + - match: { hits.total: 1 } + +--- +"Update-by-query with runas user with minimal privileges works": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: minimal_user} + update-by-query: + refresh: true + index: source + body: + script: + inline: ctx._source['hi'] = 'there' + - match: {updated: 1} + + - do: + search: + index: source + body: + query: + match: + hi: there + - match: { hits.total: 1 } + +--- +"Update-by-query as readonly user is forbidden": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: readonly_user} + catch: forbidden + update-by-query: + index: source + +--- +"Update-by-query as user that can't read from the source is forbidden": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: dest_only_user} + catch: forbidden + update-by-query: + index: source + +--- +"Update-by-query misses hidden docs": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + index: + index: source + type: foo + id: 2 + body: { "text": "test", "hidden": true } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_docs_user} + update-by-query: + refresh: true + index: source + body: + script: + inline: ctx._source['hi'] = 'there' + - match: {updated: 1} + + # We only updated one doc, presumably the one without the hidden field + - do: + search: + index: source + body: + query: + match: + hi: there + - match: { hits.total: 1 } + + # We didn't update the doc with the hidden field + - do: + search: + index: source + body: + query: + bool: + must: + - match: + hi: there + - match: + hidden: true + - match: { hits.total: 0 } + +--- +"Reindex misses hidden fields": + + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test", "foo": "z", "bar": "z" } + - do: + indices.refresh: {} + + - do: + headers: {es-shield-runas-user: can_not_see_hidden_fields_user} + update-by-query: + index: source + body: + script: + inline: ctx._source['hi'] = ctx._source['text'] + ';' + ctx._source['foo'] + - match: {updated: 1} + + - do: + get: + index: source + type: foo + id: 1 + # These were visible to the user running the update-by-query so they stayed. + - match: { _source.foo: z } + - match: { _source.bar: z } + # This wasn't visible to the update-by-query-ing user so it is gone. + - is_false: _source.text + # The reindexing user tried to sneak an invisible field using a script and got a null for their trouble. + - match: { _source.hi: null;z }