security: Added `set_security_user` ingest processor that enriches documents with user details of the current authenticated user

This is useful if an index is shared with many small customers, which are to small to have their own index or shard,
 and in order to share an index safely they will need to use document level security. This processor can then automatically
 add the username or roles of the current authenticated user to the documents being indexed, so that the DLS query can be simple. (`username: abc` only return data inserted by user abc)

Closes elastic/elasticsearch#2738

Original commit: elastic/x-pack-elasticsearch@f4df2f6d6f
This commit is contained in:
Martijn van Groningen 2016-07-14 18:07:16 +02:00
parent 7bb4c613eb
commit cc7cfb7fd9
8 changed files with 534 additions and 2 deletions

View File

@ -40,7 +40,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexModule;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.IngestPlugin;
import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.watcher.ResourceWatcherService;
@ -89,6 +91,7 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.authz.AuthorizationModule; import org.elasticsearch.xpack.security.authz.AuthorizationModule;
import org.elasticsearch.xpack.security.authz.InternalAuthorizationService; import org.elasticsearch.xpack.security.authz.InternalAuthorizationService;
import org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor;
import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache; import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
import org.elasticsearch.xpack.security.authz.accesscontrol.SecurityIndexSearcherWrapper; import org.elasticsearch.xpack.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
import org.elasticsearch.xpack.security.authz.store.FileRolesStore; import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
@ -125,7 +128,7 @@ import static java.util.Collections.singletonList;
/** /**
* *
*/ */
public class Security implements ActionPlugin { public class Security implements ActionPlugin, IngestPlugin {
private static final ESLogger logger = Loggers.getLogger(XPackPlugin.class); private static final ESLogger logger = Loggers.getLogger(XPackPlugin.class);
@ -458,6 +461,11 @@ public class Security implements ActionPlugin {
RestChangePasswordAction.class); RestChangePasswordAction.class);
} }
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
}
public void onModule(NetworkModule module) { public void onModule(NetworkModule module) {
if (transportClientMode) { if (transportClientMode) {

View File

@ -77,6 +77,10 @@ public class Authentication {
return deserializeHeaderAndPutInContext(authenticationHeader, ctx, cryptoService, sign); return deserializeHeaderAndPutInContext(authenticationHeader, ctx, cryptoService, sign);
} }
public static Authentication getAuthentication(ThreadContext context) {
return context.getTransient(Authentication.AUTHENTICATION_KEY);
}
static Authentication deserializeHeaderAndPutInContext(String header, ThreadContext ctx, CryptoService cryptoService, boolean sign) static Authentication deserializeHeaderAndPutInContext(String header, ThreadContext ctx, CryptoService cryptoService, boolean sign)
throws IOException, IllegalArgumentException { throws IOException, IllegalArgumentException {
assert ctx.getTransient(AUTHENTICATION_KEY) == null; assert ctx.getTransient(AUTHENTICATION_KEY) == null;

View File

@ -0,0 +1,150 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.authz.accesscontrol;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.xpack.security.authc.Authentication;
import org.elasticsearch.xpack.security.user.User;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
/**
* A processor that adds information of the current authenticated user to the document being ingested.
*/
public final class SetSecurityUserProcessor extends AbstractProcessor {
public static final String TYPE = "set_security_user";
private final ThreadContext threadContext;
private final String field;
private final Set<Property> properties;
SetSecurityUserProcessor(String tag, ThreadContext threadContext, String field, Set<Property> properties) {
super(tag);
this.threadContext = threadContext;
this.field = field;
this.properties = properties;
}
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
Authentication authentication = Authentication.getAuthentication(threadContext);
if (authentication == null) {
throw new IllegalStateException("No user authenticated, only use this processor via authenticated user");
}
User user = authentication.getUser();
if (user == null) {
throw new IllegalStateException("No user for authentication");
}
Map<String, Object> userObject = new HashMap<>();
for (Property property : properties) {
switch (property) {
case USERNAME:
if (user.principal() != null) {
userObject.put("username", user.principal());
}
break;
case FULL_NAME:
if (user.fullName() != null) {
userObject.put("full_name", user.fullName());
}
break;
case EMAIL:
if (user.email() != null) {
userObject.put("email", user.email());
}
break;
case ROLES:
if (user.roles() != null && user.roles().length != 0) {
userObject.put("roles", Arrays.asList(user.roles()));
}
break;
case METADATA:
if (user.metadata() != null && user.metadata().isEmpty() == false) {
userObject.put("metadata", user.metadata());
}
break;
default:
throw new UnsupportedOperationException("unsupported property [" + property + "]");
}
}
ingestDocument.setFieldValue(field, userObject);
}
@Override
public String getType() {
return TYPE;
}
String getField() {
return field;
}
Set<Property> getProperties() {
return properties;
}
public static final class Factory implements Processor.Factory {
private final ThreadContext threadContext;
public Factory(ThreadContext threadContext) {
this.threadContext = threadContext;
}
@Override
public SetSecurityUserProcessor create(Map<String, Processor.Factory> processorFactories, String tag,
Map<String, Object> config) throws Exception {
String field = readStringProperty(TYPE, tag, config, "field");
List<String> propertyNames = readOptionalList(TYPE, tag, config, "properties");
Set<Property> properties;
if (propertyNames != null) {
properties = EnumSet.noneOf(Property.class);
for (String propertyName : propertyNames) {
properties.add(Property.parse(tag, propertyName));
}
} else {
properties = EnumSet.allOf(Property.class);
}
return new SetSecurityUserProcessor(tag, threadContext, field, properties);
}
}
enum Property {
USERNAME,
FULL_NAME,
EMAIL,
ROLES,
METADATA;
static Property parse(String tag, String value) {
try {
return valueOf(value.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
// not using the original exception as its message is confusing
// (e.g. 'No enum constant org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor.Property.INVALID')
throw newConfigurationException(TYPE, tag, "properties", "Property value [" + value + "] is in valid");
}
}
}
}

View File

@ -0,0 +1,60 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.authz.accesscontrol;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor.Property;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
public class SetSecurityUserProcessorFactoryTests extends ESTestCase {
public void testProcessor() throws Exception {
SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
SetSecurityUserProcessor processor = factory.create(null, "_tag", config);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(EnumSet.allOf(Property.class)));
}
public void testProcessor_noField() throws Exception {
SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null);
Map<String, Object> config = new HashMap<>();
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config));
assertThat(e.getHeader("property_name").get(0), equalTo("field"));
assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag"));
}
public void testProcessor_validProperties() throws Exception {
SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("properties", Arrays.asList(Property.USERNAME.name(), Property.ROLES.name()));
SetSecurityUserProcessor processor = factory.create(null, "_tag", config);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(EnumSet.of(Property.USERNAME, Property.ROLES)));
}
public void testProcessor_invalidProperties() throws Exception {
SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("properties", Arrays.asList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config));
assertThat(e.getHeader("property_name").get(0), equalTo("properties"));
assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag"));
}
}

View File

@ -0,0 +1,150 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.authz.accesscontrol;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.security.authc.Authentication;
import org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor.Property;
import org.elasticsearch.xpack.security.user.User;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
public class SetSecurityUserProcessorTests extends ESTestCase {
public void testProcessor() throws Exception {
User user = new User("_username", new String[]{"role1", "role2"}, "firstname lastname", "_email",
Collections.singletonMap("key", "value"));
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(5));
assertThat(result.get("username"), equalTo("_username"));
assertThat(((List) result.get("roles")).size(), equalTo(2));
assertThat(((List) result.get("roles")).get(0), equalTo("role1"));
assertThat(((List) result.get("roles")).get(1), equalTo("role2"));
assertThat(result.get("full_name"), equalTo("firstname lastname"));
assertThat(result.get("email"), equalTo("_email"));
assertThat(((Map) result.get("metadata")).size(), equalTo(1));
assertThat(((Map) result.get("metadata")).get("key"), equalTo("value"));
// test when user holds no data:
threadContext = new ThreadContext(Settings.EMPTY);
user = new User(null, null, null);
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(0));
}
public void testNoCurrentUser() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.allOf(Property.class));
IllegalStateException e = expectThrows(IllegalStateException.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("No user authenticated, only use this processor via authenticated user"));
}
public void testUsernameProperties() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
User user = new User("_username", null, null);
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.of(Property.USERNAME));
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(1));
assertThat(result.get("username"), equalTo("_username"));
}
public void testRolesProperties() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
User user = new User(null, "role1", "role2");
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.of(Property.ROLES));
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(1));
assertThat(((List) result.get("roles")).size(), equalTo(2));
assertThat(((List) result.get("roles")).get(0), equalTo("role1"));
assertThat(((List) result.get("roles")).get(1), equalTo("role2"));
}
public void testFullNameProperties() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
User user = new User(null, null, "_full_name", null, null);
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.of(Property.FULL_NAME));
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(1));
assertThat(result.get("full_name"), equalTo("_full_name"));
}
public void testEmailProperties() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
User user = new User(null, null, null, "_email", null);
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.of(Property.EMAIL));
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(1));
assertThat(result.get("email"), equalTo("_email"));
}
public void testMetadataProperties() throws Exception {
ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
User user = new User(null, null, null, null, Collections.singletonMap("key", "value"));
Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
threadContext.putTransient(Authentication.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", threadContext, "_field", EnumSet.of(Property.METADATA));
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(1));
assertThat(((Map) result.get("metadata")).size(), equalTo(1));
assertThat(((Map) result.get("metadata")).get("key"), equalTo("value"));
}
}

View File

@ -0,0 +1,10 @@
"Ingest plugin installed":
- do:
cluster.state: {}
- set: {master_node: master}
- do:
nodes.info: {}
- match: { nodes.$master.ingest.processors.0.type: set_security_user }

View File

@ -0,0 +1,143 @@
---
setup:
- skip:
features: headers
- do:
cluster.health:
wait_for_status: yellow
- do:
ingest.put_pipeline:
id: "my_pipeline"
body: >
{
"processors": [
{
"set_security_user" : {
"field" : "user",
"properties" : ["username", "roles"]
}
}
]
}
- do:
xpack.security.put_user:
username: "joe"
body: >
{
"password": "changeme",
"roles" : [ "company_x_logs_role" ]
}
- do:
xpack.security.put_user:
username: "john"
body: >
{
"password": "changeme",
"roles" : [ "company_y_logs_role" ]
}
- do:
xpack.security.put_role:
name: "company_x_logs_role"
body: >
{
"cluster": ["all"],
"indices": [
{
"names": "shared_logs",
"privileges": ["all"],
"query" : {
"term" : { "user.roles" : "company_x_logs_role" }
}
}
]
}
- do:
xpack.security.put_role:
name: "company_y_logs_role"
body: >
{
"cluster": ["all"],
"indices": [
{
"names": "shared_logs",
"privileges": ["all"],
"query" : {
"term" : { "user.roles" : "company_y_logs_role" }
}
}
]
}
---
teardown:
- do:
xpack.security.delete_user:
username: "joe"
ignore: 404
- do:
xpack.security.delete_user:
username: "john"
ignore: 404
- do:
xpack.security.delete_role:
name: "company_x_logs_role"
ignore: 404
- do:
xpack.security.delete_role:
name: "company_y_logs_role"
ignore: 404
---
"Test shared index seperating user by using DLS":
- do:
headers:
Authorization: "Basic am9lOmNoYW5nZW1l"
index:
index: shared_logs
type: type
id: 1
pipeline: "my_pipeline"
body: >
{
"log": "Joe's first log entry"
}
- do:
headers:
Authorization: "Basic am9objpjaGFuZ2VtZQ=="
index:
index: shared_logs
type: type
id: 2
pipeline: "my_pipeline"
body: >
{
"log": "John's first log entry"
}
- do:
indices.refresh: {}
# Joe searches:
- do:
headers:
Authorization: "Basic am9lOmNoYW5nZW1l"
search:
index: shared_logs
body: { "query" : { "match_all" : {} } }
- match: { hits.total: 1}
- match: { hits.hits.0._source.user.username: joe}
- match: { hits.hits.0._source.user.roles.0: company_x_logs_role}
# John searches:
- do:
headers:
Authorization: "Basic am9objpjaGFuZ2VtZQ=="
search:
index: shared_logs
body: { "query" : { "match_all" : {} } }
- match: { hits.total: 1}
- match: { hits.hits.0._source.user.username: john}
- match: { hits.hits.0._source.user.roles.0: company_y_logs_role}

View File

@ -35,8 +35,10 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexModule;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.license.plugin.Licensing; import org.elasticsearch.license.plugin.Licensing;
import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.IngestPlugin;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHandler;
@ -75,7 +77,7 @@ import org.elasticsearch.xpack.support.clock.SystemClock;
import org.elasticsearch.xpack.watcher.Watcher; import org.elasticsearch.xpack.watcher.Watcher;
import org.elasticsearch.xpack.watcher.support.WatcherScript; import org.elasticsearch.xpack.watcher.support.WatcherScript;
public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin { public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin {
public static final String NAME = "x-pack"; public static final String NAME = "x-pack";
@ -308,6 +310,11 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin {
return handlers; return handlers;
} }
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return security.getProcessors(parameters);
}
public void onModule(AuthenticationModule module) { public void onModule(AuthenticationModule module) {
if (extensionsService != null) { if (extensionsService != null) {
extensionsService.onModule(module); extensionsService.onModule(module);