honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Init beats monitoring support (elastic/x-pack-elasticsearch#543) 

- introduce builtin user 'beats_system'
- init monitoring-beats ES mapping
- add beats to MonitoredSystem + Resolver

Original commit: elastic/x-pack-elasticsearch@4d7b45d54d
This commit is contained in:
Steffen Siering 2017-03-13 10:29:42 -07:00 committed by GitHub
parent e5cc038616
commit 07bde45671
16 changed files with 270 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java
View File

@ -11,7 +11,8 @@ public enum MonitoredSystem {
ES("es"),
KIBANA("kibana"),
LOGSTASH("logstash");
LOGSTASH("logstash"),
BEATS("beats");
private final String system;
@ -31,6 +32,8 @@ public enum MonitoredSystem {
return KIBANA;
case "logstash":
return LOGSTASH;
case "beats":
return BEATS;
default:
throw new IllegalArgumentException("Unknown monitoring system [" + system + "]");
}

2
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtils.java
View File

@ -24,7 +24,7 @@ public final class MonitoringTemplateUtils {
/**
* Data types that should be supported by the {@linkplain #DATA_INDEX data index} that were not by the initial release.
*/
public static final String[] NEW_DATA_TYPES = { "kibana", "logstash" };
public static final String[] NEW_DATA_TYPES = { "kibana", "logstash", "beats" };
private MonitoringTemplateUtils() {
}

1
plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java
View File

@ -51,6 +51,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
// register resolvers for monitored systems
registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings);
registerMonitoredSystem(MonitoredSystem.BEATS, settings);
}
/**

32
plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmMigrator.java
View File

@ -24,6 +24,8 @@ import org.elasticsearch.xpack.security.SecurityLifecycleService;
import org.elasticsearch.xpack.security.authc.support.Hasher;
import org.elasticsearch.xpack.security.authc.support.SecuredString;
import org.elasticsearch.xpack.security.client.SecurityClient;
import org.elasticsearch.xpack.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.security.user.BuiltinUserInfo;
import org.elasticsearch.xpack.security.user.LogstashSystemUser;
import org.elasticsearch.xpack.security.user.User;
@ -37,6 +39,8 @@ import java.util.function.BiConsumer;
import static java.util.Collections.emptyList;
import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME;
import java.util.ArrayList;
/**
* Performs migration steps for the {@link NativeRealm} and {@link ReservedRealm}.
* When upgrading an Elasticsearch/X-Pack installation from a previous version, this class is responsible for ensuring that user/role
@ -47,6 +51,10 @@ public class NativeRealmMigrator {
private final XPackLicenseState licenseState;
private final Logger logger;
private InternalClient client;
private final BuiltinUserInfo[] builtinUsers = new BuiltinUserInfo[] {
LogstashSystemUser.USER_INFO,
BeatsSystemUser.USER_INFO
};
public NativeRealmMigrator(Settings settings, XPackLicenseState licenseState, InternalClient internalClient) {
this.licenseState = licenseState;
@ -85,9 +93,12 @@ public class NativeRealmMigrator {
private List<BiConsumer<Version, ActionListener<Void>>> collectUpgradeTasks(@Nullable Version previousVersion) {
List<BiConsumer<Version, ActionListener<Void>>> tasks = new ArrayList<>();
if (shouldDisableLogstashUser(previousVersion)) {
tasks.add(this::createLogstashUserAsDisabled);
for (BuiltinUserInfo info : builtinUsers) {
if (isNewUser(previousVersion, info)) {
tasks.add((v,l) -> createUserAsDisabled(info, v, l));
}
}
if (shouldConvertDefaultPasswords(previousVersion)) {
tasks.add(this::doConvertDefaultPasswords);
}
@ -95,34 +106,35 @@ public class NativeRealmMigrator {
}
/**
* If we're upgrading from a security version where the {@link LogstashSystemUser} did not exist, then we mark the user as disabled.
* If we're upgrading from a security version where the new user did not exist, then we mark the user as disabled.
* Otherwise the user will exist with a default password, which is desirable for an <em>out-of-the-box</em> experience in fresh
* installs but problematic for already-locked-down upgrades.
*/
private boolean shouldDisableLogstashUser(@Nullable Version previousVersion) {
return previousVersion != null && previousVersion.before(LogstashSystemUser.DEFINED_SINCE);
private boolean isNewUser(@Nullable Version previousVersion, BuiltinUserInfo info) {
return previousVersion != null && previousVersion.before(info.getDefinedSince());
}
private void createLogstashUserAsDisabled(@Nullable Version previousVersion, ActionListener<Void> listener) {
private void createUserAsDisabled(BuiltinUserInfo info, @Nullable Version previousVersion, ActionListener<Void> listener) {
logger.info("Upgrading security from version [{}] - new reserved user [{}] will default to disabled",
previousVersion, LogstashSystemUser.NAME);
previousVersion, info.getName());
// Only clear the cache is authentication is allowed by the current license
// otherwise the license management checks will prevent it from completing successfully.
final boolean clearCache = licenseState.isAuthAllowed();
client.prepareGet(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, LogstashSystemUser.NAME).execute(
final String userName = info.getName();
client.prepareGet(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, userName).execute(
ActionListener.wrap(getResponse -> {
if (getResponse.isExists()) {
// the document exists - we shouldn't do anything
listener.onResponse(null);
} else {
client.prepareIndex(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, LogstashSystemUser.NAME)
client.prepareIndex(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, userName)
.setSource(Requests.INDEX_CONTENT_TYPE, User.Fields.ENABLED.getPreferredName(), false,
User.Fields.PASSWORD.getPreferredName(), "")
.setCreate(true)
.execute(ActionListener.wrap(r -> {
if (clearCache) {
new SecurityClient(client).prepareClearRealmCache()
.usernames(LogstashSystemUser.NAME)
.usernames(userName)
.execute(ActionListener.wrap(re -> listener.onResponse(null), listener::onFailure));
} else {
listener.onResponse(null);

9
plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.support.Exceptions;
import org.elasticsearch.xpack.security.user.AnonymousUser;
import org.elasticsearch.xpack.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.security.user.ElasticUser;
import org.elasticsearch.xpack.security.user.KibanaUser;
import org.elasticsearch.xpack.security.user.LogstashSystemUser;
@ -141,6 +142,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
case ElasticUser.NAME:
case KibanaUser.NAME:
case LogstashSystemUser.NAME:
case BeatsSystemUser.NAME:
return XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings);
default:
return AnonymousUser.isAnonymousUsername(username, settings);
@ -156,6 +158,8 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
return new KibanaUser(userInfo.enabled);
case LogstashSystemUser.NAME:
return new LogstashSystemUser(userInfo.enabled);
case BeatsSystemUser.NAME:
return new BeatsSystemUser(userInfo.enabled);
default:
if (anonymousEnabled && anonymousUser.principal().equals(username)) {
return anonymousUser;
@ -181,6 +185,9 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
userInfo = reservedUserInfos.get(LogstashSystemUser.NAME);
users.add(new LogstashSystemUser(userInfo == null || userInfo.enabled));
userInfo = reservedUserInfos.get(BeatsSystemUser.NAME);
users.add(new BeatsSystemUser(userInfo == null || userInfo.enabled));
if (anonymousEnabled) {
users.add(anonymousUser);
}
@ -223,6 +230,8 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
switch (username) {
case LogstashSystemUser.NAME:
return LogstashSystemUser.DEFINED_SINCE;
case BeatsSystemUser.NAME:
return BeatsSystemUser.DEFINED_SINCE;
default:
return Version.V_5_0_0;
}

2
plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java
View File

@ -60,6 +60,8 @@ public class ReservedRolesStore {
null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.put("logstash_system", new RoleDescriptor("logstash_system", new String[] { "monitor", MonitoringBulkAction.NAME},
null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.put("beats_system", new RoleDescriptor("beats_system", new String[] { "monitor", MonitoringBulkAction.NAME},
null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.immutableMap();
}

20
plugin/src/main/java/org/elasticsearch/xpack/security/user/BeatsSystemUser.java Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.user;
import org.elasticsearch.Version;
import org.elasticsearch.xpack.security.support.MetadataUtils;
public class BeatsSystemUser extends User {
public static final String NAME = "beats_system";
private static final String ROLE_NAME = "beats_system";
public static final Version DEFINED_SINCE = Version.V_6_0_0_alpha1_UNRELEASED;
public static final BuiltinUserInfo USER_INFO = new BuiltinUserInfo(NAME, ROLE_NAME, DEFINED_SINCE);
public BeatsSystemUser(boolean enabled) {
super(NAME, new String[]{ ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);
}
}

38
plugin/src/main/java/org/elasticsearch/xpack/security/user/BuiltinUserInfo.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.security.user;
import org.elasticsearch.Version;
/**
* BuiltinUserInfo provides common user meta data for newly introduced pre defined System Users.
*/
public class BuiltinUserInfo {
private final String name;
private final String role;
private final Version definedSince;
public BuiltinUserInfo(String name, String role, Version definedSince) {
this.name = name;
this.role = role;
this.definedSince = definedSince;
}
/** Get the builtin users name. */
public String getName() {
return name;
}
/** Get the builtin users default role name. */
public String getRole() {
return role;
}
/** Get version the builtin user was introduced with. */
public Version getDefinedSince() {
return definedSince;
}
}

1
plugin/src/main/java/org/elasticsearch/xpack/security/user/LogstashSystemUser.java
View File

@ -16,6 +16,7 @@ public class LogstashSystemUser extends User {
public static final String NAME = "logstash_system";
private static final String ROLE_NAME = "logstash_system";
public static final Version DEFINED_SINCE = Version.V_5_2_0_UNRELEASED;
public static final BuiltinUserInfo USER_INFO = new BuiltinUserInfo(NAME, ROLE_NAME, DEFINED_SINCE);
public LogstashSystemUser(boolean enabled) {
super(NAME, new String[]{ ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);

88
plugin/src/main/resources/monitoring-beats.json Normal file
View File

@ -0,0 +1,88 @@
{
"template": ".monitoring-beats-${monitoring.template.version}-*",
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
},
"mappings": {
"beats_stats": {
"properties": {
"cluster_uuid": {
"type": "keyword"
},
"timestamp": {
"type": "date",
"format": "date_time"
},
"source_node": {
"properties": {
"uuid": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"transport_address": {
"type": "keyword"
},
"ip": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
},
"beats_stats": {
"properties": {
"timestamp": {
"type": "date",
"format": "date_time"
},
"tags": {
"type": "keyword"
},
"beat": {
"properties": {
"uuid": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"name": {
"type": "keyword"
}
}
},
"metrics": {
"properties": {
}
}
}
}
}
}
}
}

3
plugin/src/main/resources/monitoring-data.json
View File

@ -23,6 +23,9 @@
},
"logstash": {
"enabled": false
},
"beats": {
"enabled": false
}
}
}

2
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
View File

@ -43,7 +43,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
* kibana, logstash, beats
*/
private final int EXPECTED_TYPES = MonitoringTemplateUtils.NEW_DATA_TYPES.length;
private final int EXPECTED_TEMPLATES = 4;
private final int EXPECTED_TEMPLATES = 5;
private final RestClient client = mock(RestClient.class);
private final Response versionResponse = mock(Response.class);

2
plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java
View File

@ -307,7 +307,7 @@ public class HttpExporterTests extends ESTestCase {
equalTo(version + typeMappings.size() + templates.size() + pipelines.size() + bwc.size()));
assertThat(version, equalTo(1));
assertThat(typeMappings, hasSize(MonitoringTemplateUtils.NEW_DATA_TYPES.length));
assertThat(templates, hasSize(4));
assertThat(templates, hasSize(5));
assertThat(pipelines, hasSize(useIngest ? 1 : 0));
assertThat(bwc, hasSize(1));

46
plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmMigratorTests.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.xpack.security.SecurityLifecycleService;
import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheAction;
import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest;
import org.elasticsearch.xpack.security.authc.support.Hasher;
import org.elasticsearch.xpack.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.security.user.ElasticUser;
import org.elasticsearch.xpack.security.user.KibanaUser;
import org.elasticsearch.xpack.security.user.LogstashSystemUser;
@ -150,14 +151,14 @@ public class NativeRealmMigratorTests extends ESTestCase {
}
public void testNoChangeOnFreshInstall() throws Exception {
verifyUpgrade(null, false, false);
verifyUpgrade(null, null, false);
}
public void testNoChangeOnUpgradeAfterV5_3() throws Exception {
verifyUpgrade(randomFrom(Version.V_6_0_0_alpha1_UNRELEASED), false, false);
verifyUpgrade(randomFrom(Version.V_6_0_0_alpha1_UNRELEASED), null, false);
}
public void testDisableLogstashAndConvertPasswordsOnUpgradeFromVersionPriorToV5_2() throws Exception {
public void testDisableLogstashBeatsAndConvertPasswordsOnUpgradeFromVersionPriorToV5_2() throws Exception {
this.reservedUsers = Collections.singletonMap(
KibanaUser.NAME,
MapBuilder.<String, Object>newMapBuilder()
@ -165,7 +166,21 @@ public class NativeRealmMigratorTests extends ESTestCase {
.put(User.Fields.ENABLED.getPreferredName(), false)
.immutableMap()
);
verifyUpgrade(randomFrom(Version.V_5_1_1_UNRELEASED, Version.V_5_0_2, Version.V_5_0_0), true, true);
String[] disabledUsers = new String[]{LogstashSystemUser.NAME, BeatsSystemUser.NAME};
verifyUpgrade(randomFrom(Version.V_5_1_1_UNRELEASED, Version.V_5_0_2, Version.V_5_0_0), disabledUsers, true);
}
public void testDisableBeatsAndConvertPasswordsOnUpgradeFromVersionPriorToV6() throws Exception {
this.reservedUsers = Collections.singletonMap(
KibanaUser.NAME,
MapBuilder.<String, Object>newMapBuilder()
.put(User.Fields.PASSWORD.getPreferredName(), new String(Hasher.BCRYPT.hash(ReservedRealm.DEFAULT_PASSWORD_TEXT)))
.put(User.Fields.ENABLED.getPreferredName(), false)
.immutableMap()
);
String[] disabledUsers = new String[]{BeatsSystemUser.NAME};
Version version = randomFrom(Version.V_5_3_0_UNRELEASED, Version.V_5_2_1_UNRELEASED);
verifyUpgrade(version, disabledUsers, true);
}
public void testConvertPasswordsOnUpgradeFromVersion5_2() throws Exception {
@ -177,24 +192,31 @@ public class NativeRealmMigratorTests extends ESTestCase {
.put(User.Fields.ENABLED.getPreferredName(), randomBoolean())
.immutableMap()
));
verifyUpgrade(Version.V_5_2_0_UNRELEASED, false, true);
String[] disabledUsers = new String[]{BeatsSystemUser.NAME};
verifyUpgrade(Version.V_5_2_0_UNRELEASED, disabledUsers, true);
}
private void verifyUpgrade(Version fromVersion, boolean disableLogstashUser, boolean convertDefaultPasswords) throws Exception {
private void verifyUpgrade(Version fromVersion, String[] disabledUsers, boolean convertDefaultPasswords) throws Exception {
final PlainActionFuture<Boolean> future = doUpgrade(fromVersion);
boolean expectedResult = false;
if (disableLogstashUser) {
if (disabledUsers != null) {
final int userCount = disabledUsers.length;
final boolean clearCache = licenseState.isAuthAllowed();
ArgumentCaptor<GetRequest> captor = ArgumentCaptor.forClass(GetRequest.class);
verify(mockClient).execute(eq(GetAction.INSTANCE), captor.capture(), any(ActionListener.class));
assertEquals(LogstashSystemUser.NAME, captor.getValue().id());
verify(mockClient, times(userCount)).execute(eq(GetAction.INSTANCE), captor.capture(), any(ActionListener.class));
for (int i = 0; i < userCount; i++) {
assertEquals(disabledUsers[i], captor.getAllValues().get(i).id());
}
ArgumentCaptor<IndexRequest> indexCaptor = ArgumentCaptor.forClass(IndexRequest.class);
verify(mockClient).execute(eq(IndexAction.INSTANCE), indexCaptor.capture(), any(ActionListener.class));
assertEquals(LogstashSystemUser.NAME, indexCaptor.getValue().id());
verify(mockClient, times(userCount)).execute(eq(IndexAction.INSTANCE), indexCaptor.capture(), any(ActionListener.class));
for (int i = 0; i < userCount; i++) {
assertEquals(disabledUsers[i], captor.getAllValues().get(i).id());
assertEquals(false, indexCaptor.getValue().sourceAsMap().get(User.Fields.ENABLED.getPreferredName()));
}
if (clearCache) {
verify(mockClient).execute(eq(ClearRealmCacheAction.INSTANCE), any(ClearRealmCacheRequest.class),
verify(mockClient, times(userCount)).execute(eq(ClearRealmCacheAction.INSTANCE), any(ClearRealmCacheRequest.class),
any(ActionListener.class));
}
expectedResult = true;

4
plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java
View File

@ -18,6 +18,7 @@ import org.elasticsearch.xpack.security.authc.support.Hasher;
import org.elasticsearch.xpack.security.authc.support.SecuredString;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.user.AnonymousUser;
import org.elasticsearch.xpack.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.security.user.ElasticUser;
import org.elasticsearch.xpack.security.user.KibanaUser;
import org.elasticsearch.xpack.security.user.LogstashSystemUser;
@ -296,7 +297,8 @@ public class ReservedRealmTests extends ESTestCase {
new AnonymousUser(Settings.EMPTY), securityLifecycleService);
PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
reservedRealm.users(userFuture);
assertThat(userFuture.actionGet(), containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true)));
assertThat(userFuture.actionGet(), containsInAnyOrder(new ElasticUser(true), new KibanaUser(true),
new LogstashSystemUser(true), new BeatsSystemUser(true)));
}
public void testGetUsersDisabled() {

22
plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStoreTests.java
View File

@ -339,4 +339,26 @@ public class ReservedRolesStoreTests extends ESTestCase {
assertThat(logstashSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAsciiOfLengthBetween(8, 24)),
is(false));
}
public void testBeatsSystemRole() {
RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("beats_system");
assertNotNull(roleDescriptor);
assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
Role beatsSystemRole = Role.builder(roleDescriptor, null).build();
assertThat(beatsSystemRole.cluster().check(ClusterHealthAction.NAME), is(true));
assertThat(beatsSystemRole.cluster().check(ClusterStateAction.NAME), is(true));
assertThat(beatsSystemRole.cluster().check(ClusterStatsAction.NAME), is(true));
assertThat(beatsSystemRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
assertThat(beatsSystemRole.cluster().check(ClusterRerouteAction.NAME), is(false));
assertThat(beatsSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
assertThat(beatsSystemRole.cluster().check(MonitoringBulkAction.NAME), is(true));
assertThat(beatsSystemRole.runAs().check(randomAsciiOfLengthBetween(1, 30)), is(false));
assertThat(beatsSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
assertThat(beatsSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
assertThat(beatsSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAsciiOfLengthBetween(8, 24)),
is(false));
}
}