honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-27 02:18:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose more authentication info to ingest pipeline (#51305) (#52119)

Browse Source 
The changes add more granularity for identiying the data ingestion user.
The ingest pipeline can now be configure to record authentication realm and
type. It can also record API key name and ID when one is in use. 
This improves traceability when data are being ingested from multiple agents
and will become more relevant with the incoming support of required
pipelines (#46847)

Resolves: #49106
This commit is contained in:
Yang Wang 2020-02-11 23:05:01 +11:00 committed by GitHub
parent 343ced42be
commit 16ba59e9d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 222 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/reference/ingest/processors/set-security-user.asciidoc
View File

@ -1,8 +1,15 @@
[[ingest-node-set-security-user-processor]]
=== Set Security User Processor
Sets user-related details (such as `username`, `roles`, `email`, `full_name`
and `metadata` ) from the current
Sets user-related details (such as `username`, `roles`, `email`, `full_name`,
`metadata`, `api_key`, `realm` and `authentication_type`) from the current
authenticated user to the current document by pre-processing the ingest.
The `api_key` property exists only if the user authenticates with an
API key. It is an object containing the `id` and `name` fields of the API key.
The `realm` property is also an object with two fields, `name` and `type`.
When using API key authentication, the `realm` property refers to the realm
from which the API key is created.
The `authentication_type` property is a string that can take value from
`REALM`, `API_KEY`, `TOKEN` and `ANONYMOUS`.
IMPORTANT: Requires an authenticated user for the index request.
@ -10,9 +17,9 @@ IMPORTANT: Requires an authenticated user for the index request.
.Set Security User Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to store the user information into.
| `properties` | no | [`username`, `roles`, `email`, `full_name`, `metadata`] | Controls what user related properties are added to the `field`.
| Name | Required | Default | Description
| `field` | yes | - | The field to store the user information into.
| `properties` | no | [`username`, `roles`, `email`, `full_name`, `metadata`, `api_key`, `realm`, `authentication_type`] | Controls what user related properties are added to the `field`.
include::common-options.asciidoc[]
|======

4
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
View File

@ -82,6 +82,10 @@ public class Authentication implements ToXContentObject {
return lookedUpBy;
}
public RealmRef getSourceRealm() {
return lookedUpBy == null ? authenticatedBy : lookedUpBy;
}
public Version getVersion() {
return version;
}

3
x-pack/plugin/core/src/main/resources/security-index-template-7.json
View File

@ -194,6 +194,9 @@
},
"realm" : {
"type" : "keyword"
},
"realm_type" : {
"type" : "keyword"
}
}
},

33
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java Normal file
View File

@ -0,0 +1,33 @@
/*
*
* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* * or more contributor license agreements. Licensed under the Elastic License;
* * you may not use this file except in compliance with the Elastic License.
*
*/
package org.elasticsearch.xpack.core.security.authc;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.security.user.User;
public class AuthenticationTests extends ESTestCase {
public void testWillGetLookedUpByWhenItExists() {
final Authentication.RealmRef authenticatedBy = new Authentication.RealmRef("auth_by", "auth_by_type", "node");
final Authentication.RealmRef lookedUpBy = new Authentication.RealmRef("lookup_by", "lookup_by_type", "node");
final Authentication authentication = new Authentication(
new User("user"), authenticatedBy, lookedUpBy);
assertEquals(lookedUpBy, authentication.getSourceRealm());
}
public void testWillGetAuthenticateByWhenLookupIsNull() {
final Authentication.RealmRef authenticatedBy = new Authentication.RealmRef("auth_by", "auth_by_type", "node");
final Authentication authentication = new Authentication(
new User("user"), authenticatedBy, null);
assertEquals(authenticatedBy, authentication.getSourceRealm());
}
}

14
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
View File

@ -108,9 +108,11 @@ public class ApiKeyService {
private static final Logger logger = LogManager.getLogger(ApiKeyService.class);
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
public static final String API_KEY_ID_KEY = "_security_api_key_id";
public static final String API_KEY_NAME_KEY = "_security_api_key_name";
public static final String API_KEY_REALM_NAME = "_es_api_key";
public static final String API_KEY_REALM_TYPE = "_es_api_key";
public static final String API_KEY_CREATOR_REALM = "_security_api_key_creator_realm";
public static final String API_KEY_CREATOR_REALM_NAME = "_security_api_key_creator_realm_name";
public static final String API_KEY_CREATOR_REALM_TYPE = "_security_api_key_creator_realm_type";
static final String API_KEY_ROLE_DESCRIPTORS_KEY = "_security_api_key_role_descriptors";
static final String API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY = "_security_api_key_limited_by_role_descriptors";
@ -279,8 +281,8 @@ public class ApiKeyService {
.startObject("creator")
.field("principal", authentication.getUser().principal())
.field("metadata", authentication.getUser().metadata())
.field("realm", authentication.getLookedUpBy() == null ?
authentication.getAuthenticatedBy().getName() : authentication.getLookedUpBy().getName())
.field("realm", authentication.getSourceRealm().getName())
.field("realm_type", authentication.getSourceRealm().getType())
.endObject()
.endObject();
@ -509,10 +511,12 @@ public class ApiKeyService {
: limitedByRoleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY);
final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true);
final Map<String, Object> authResultMetadata = new HashMap<>();
authResultMetadata.put(API_KEY_CREATOR_REALM, creator.get("realm"));
authResultMetadata.put(API_KEY_CREATOR_REALM_NAME, creator.get("realm"));
authResultMetadata.put(API_KEY_CREATOR_REALM_TYPE, creator.get("realm_type"));
authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors);
authResultMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleDescriptors);
authResultMetadata.put(API_KEY_ID_KEY, credentials.getId());
authResultMetadata.put(API_KEY_NAME_KEY, source.get("name"));
listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata));
} else {
listener.onResponse(AuthenticationResult.unsuccessful("api key is expired", null));
@ -886,7 +890,7 @@ public class ApiKeyService {
*/
public static String getCreatorRealmName(final Authentication authentication) {
if (authentication.getAuthenticatedBy().getType().equals(API_KEY_REALM_TYPE)) {
return (String) authentication.getMetadata().get(API_KEY_CREATOR_REALM);
return (String) authentication.getMetadata().get(API_KEY_CREATOR_REALM_NAME);
} else {
return authentication.getAuthenticatedBy().getName();
}

54
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.ingest.Processor;
import org.elasticsearch.xpack.core.security.SecurityContext;
import org.elasticsearch.xpack.core.security.authc.Authentication;
import org.elasticsearch.xpack.core.security.user.User;
import org.elasticsearch.xpack.security.authc.ApiKeyService;
import java.util.Arrays;
import java.util.EnumSet;
@ -88,6 +89,54 @@ public final class SetSecurityUserProcessor extends AbstractProcessor {
userObject.put("metadata", user.metadata());
}
break;
case API_KEY:
final String apiKey = "api_key";
final Object existingApiKeyField = userObject.get(apiKey);
@SuppressWarnings("unchecked")
final Map<String, Object> apiKeyField =
existingApiKeyField instanceof Map ? (Map<String, Object>) existingApiKeyField : new HashMap<>();
Object apiKeyName = authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY);
if (apiKeyName != null) {
apiKeyField.put("name", apiKeyName);
}
Object apiKeyId = authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY);
if (apiKeyId != null) {
apiKeyField.put("id", apiKeyId);
}
if (false == apiKeyField.isEmpty()) {
userObject.put(apiKey, apiKeyField);
}
break;
case REALM:
final String realmKey = "realm";
final Object existingRealmField = userObject.get(realmKey);
@SuppressWarnings("unchecked")
final Map<String, Object> realmField =
existingRealmField instanceof Map ? (Map<String, Object>) existingRealmField : new HashMap<>();
final Object realmName, realmType;
if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
realmName = authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME);
realmType = authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_TYPE);
} else {
realmName = authentication.getSourceRealm().getName();
realmType = authentication.getSourceRealm().getType();
}
if (realmName != null) {
realmField.put("name", realmName);
}
if (realmType != null) {
realmField.put("type", realmType);
}
if (false == realmField.isEmpty()) {
userObject.put(realmKey, realmField);
}
break;
case AUTHENTICATION_TYPE:
if (authentication.getAuthenticationType() != null) {
userObject.put("authentication_type", authentication.getAuthenticationType().toString());
}
break;
default:
throw new UnsupportedOperationException("unsupported property [" + property + "]");
}
@ -141,7 +190,10 @@ public final class SetSecurityUserProcessor extends AbstractProcessor {
FULL_NAME,
EMAIL,
ROLES,
METADATA;
METADATA,
API_KEY,
REALM,
AUTHENTICATION_TYPE;
static Property parse(String tag, String value) {
try {

17
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
View File

@ -148,6 +148,10 @@ public class ApiKeyServiceTests extends ESTestCase {
assertThat(auth.getStatus(), is(AuthenticationResult.Status.SUCCESS));
assertThat(auth.getUser(), notNullValue());
assertThat(auth.getUser().principal(), is("hulk"));
assertThat(auth.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME), is("realm1"));
assertThat(auth.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_TYPE), is("native"));
assertThat(auth.getMetadata().get(ApiKeyService.API_KEY_ID_KEY), is(id));
assertThat(auth.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY), is("test"));
}
public void testAuthenticationIsSkippedIfLicenseDoesNotAllowIt() throws Exception {
@ -284,6 +288,7 @@ public class ApiKeyServiceTests extends ESTestCase {
Map<String, Object> creatorMap = new HashMap<>();
creatorMap.put("principal", "test_user");
creatorMap.put("realm", "realm1");
creatorMap.put("realm_type", "realm_type1");
creatorMap.put("metadata", Collections.emptyMap());
sourceMap.put("creator", creatorMap);
sourceMap.put("api_key_invalidated", false);
@ -302,7 +307,7 @@ public class ApiKeyServiceTests extends ESTestCase {
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(sourceMap.get("role_descriptors")));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY),
equalTo(sourceMap.get("limited_by_role_descriptors")));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM), is("realm1"));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME), is("realm1"));
sourceMap.put("expiration_time", Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli());
future = new PlainActionFuture<>();
@ -316,7 +321,7 @@ public class ApiKeyServiceTests extends ESTestCase {
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(sourceMap.get("role_descriptors")));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY),
equalTo(sourceMap.get("limited_by_role_descriptors")));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM), is("realm1"));
assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME), is("realm1"));
sourceMap.put("expiration_time", Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli());
future = new PlainActionFuture<>();
@ -561,6 +566,14 @@ public class ApiKeyServiceTests extends ESTestCase {
assertNull(cachedApiKeyHashResult);
}
public void testWillAlwaysGetAuthenticationRealmName() {
final Authentication.RealmRef authenticatedBy = new Authentication.RealmRef("auth_by", "auth_by_type", "node");
final Authentication.RealmRef lookedUpBy = new Authentication.RealmRef("lookup_by", "lookup_by_type", "node");
final Authentication authentication = new Authentication(
new User("user"), authenticatedBy, lookedUpBy);
assertEquals("auth_by", ApiKeyService.getCreatorRealmName(authentication));
}
private ApiKeyService createApiKeyService(Settings baseSettings) {
final Settings settings = Settings.builder()
.put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)

97
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
View File

@ -5,6 +5,8 @@
*/
package org.elasticsearch.xpack.security.ingest;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.ingest.IngestDocument;
@ -14,6 +16,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication;
import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
import org.elasticsearch.xpack.core.security.user.User;
import org.elasticsearch.xpack.security.authc.ApiKeyService;
import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property;
import org.junit.Before;
import org.mockito.Mockito;
@ -48,13 +51,16 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
processor.execute(ingestDocument);
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(5));
assertThat(result.size(), equalTo(7));
assertThat(result.get("username"), equalTo("_username"));
assertThat(result.get("roles"), equalTo(Arrays.asList("role1", "role2")));
assertThat(result.get("full_name"), equalTo("firstname lastname"));
assertThat(result.get("email"), equalTo("_email"));
assertThat(((Map) result.get("metadata")).size(), equalTo(1));
assertThat(((Map) result.get("metadata")).get("key"), equalTo("value"));
assertThat(((Map) result.get("realm")).get("name"), equalTo("_name"));
assertThat(((Map) result.get("realm")).get("type"), equalTo("_type"));
assertThat(result.get("authentication_type"), equalTo("REALM"));
}
public void testProcessorWithEmptyUserData() throws Exception {
@ -62,7 +68,9 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
User user = Mockito.mock(User.class);
Authentication authentication = Mockito.mock(Authentication.class);
Mockito.when(authentication.getUser()).thenReturn(user);
Mockito.when(authentication.getAuthenticatedBy()).thenReturn(new Authentication.RealmRef("_name", "_type", "_node_name"));
final Authentication.RealmRef authRealm = new Authentication.RealmRef("_name", "_type", "_node_name");
Mockito.when(authentication.getAuthenticatedBy()).thenReturn(authRealm);
Mockito.when(authentication.getSourceRealm()).thenReturn(authRealm);
Mockito.when(authentication.getAuthenticationType()).thenReturn(AuthenticationType.REALM);
Mockito.when(authentication.encode()).thenReturn(randomAlphaOfLength(24)); // don't care as long as it's not null
new AuthenticationContextSerializer().writeToContext(authentication, threadContext);
@ -70,8 +78,12 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", securityContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
Map result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(0));
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
// Still holds data for realm and authentication type
assertThat(result.size(), equalTo(2));
assertThat(((Map) result.get("realm")).get("name"), equalTo("_name"));
assertThat(((Map) result.get("realm")).get("type"), equalTo("_type"));
assertThat(result.get("authentication_type"), equalTo("REALM"));
}
public void testNoCurrentUser() throws Exception {
@ -186,4 +198,81 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
assertThat(result2.get("other"), equalTo("test"));
}
public void testApiKeyPopulation() throws Exception {
User user = new User(randomAlphaOfLengthBetween(4, 12), null, null);
Authentication.RealmRef realmRef = new Authentication.RealmRef(
ApiKeyService.API_KEY_REALM_NAME, ApiKeyService.API_KEY_REALM_TYPE, "_node_name");
final Map<String, Object> metadata = new MapBuilder<String, Object>()
.put(ApiKeyService.API_KEY_ID_KEY, "api_key_id")
.put(ApiKeyService.API_KEY_NAME_KEY, "api_key_name")
.put(ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name")
.put(ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type")
.immutableMap();
new Authentication(user, realmRef, null, Version.CURRENT, AuthenticationType.API_KEY,metadata)
.writeToContext(threadContext);
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", securityContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(4));
assertThat(((Map) result.get("api_key")).get("name"), equalTo("api_key_name"));
assertThat(((Map) result.get("api_key")).get("id"), equalTo("api_key_id"));
assertThat(((Map) result.get("realm")).get("name"), equalTo("creator_realm_name"));
assertThat(((Map) result.get("realm")).get("type"), equalTo("creator_realm_type"));
assertThat(result.get("authentication_type"), equalTo("API_KEY"));
}
public void testWillNotOverwriteExistingApiKeyAndRealm() throws Exception {
User user = new User(randomAlphaOfLengthBetween(4, 12), null, null);
Authentication.RealmRef realmRef = new Authentication.RealmRef(
ApiKeyService.API_KEY_REALM_NAME, ApiKeyService.API_KEY_REALM_TYPE, "_node_name");
final Map<String, Object> metadata =new MapBuilder<String, Object>()
.put(ApiKeyService.API_KEY_ID_KEY, "api_key_id")
.put(ApiKeyService.API_KEY_NAME_KEY, "api_key_name")
.put(ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name")
.put(ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type")
.immutableMap();
new Authentication(user, realmRef, null, Version.CURRENT, AuthenticationType.API_KEY, metadata)
.writeToContext(threadContext);
IngestDocument ingestDocument = new IngestDocument(IngestDocument.deepCopyMap(
new MapBuilder<String, Object>().put("_field",
new MapBuilder<>()
.put("api_key", new MapBuilder<>().put("version", 42).immutableMap())
.put("realm", new MapBuilder<>().put("id", 7).immutableMap()).immutableMap()
).immutableMap()), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", securityContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(4));
assertThat(((Map) result.get("api_key")).get("version"), equalTo(42));
assertThat(((Map) result.get("realm")).get("id"), equalTo(7));
}
public void testWillSetRunAsRealmForNonApiAuth() throws Exception {
User user = new User(randomAlphaOfLengthBetween(4, 12), null, null);
Authentication.RealmRef authRealmRef = new Authentication.RealmRef(
randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
Authentication.RealmRef lookedUpRealmRef = new Authentication.RealmRef(
randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
new Authentication(user, authRealmRef, lookedUpRealmRef, Version.CURRENT,
randomFrom(AuthenticationType.REALM, AuthenticationType.TOKEN, AuthenticationType.INTERNAL),
Collections.emptyMap()).writeToContext(threadContext);
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
SetSecurityUserProcessor processor = new SetSecurityUserProcessor("_tag", securityContext, "_field", EnumSet.allOf(Property.class));
processor.execute(ingestDocument);
Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
assertThat(result.size(), equalTo(3));
assertThat(((Map) result.get("realm")).get("name"), equalTo(lookedUpRealmRef.getName()));
assertThat(((Map) result.get("realm")).get("type"), equalTo(lookedUpRealmRef.getType()));
}
}