diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java index 7a2194d92a3..4fa161df6c7 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java @@ -46,12 +46,13 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust this.indexAuditTrail = indexAuditTrail; this.nativeUserStore = nativeUserStore; this.nativeRolesStore = nativeRolesStore; - // TODO: define a common interface for these and delegate from one place. nativeUserStore store is it's on cluster + // TODO: define a common interface for these and delegate from one place. nativeUserStore store is it's on + // cluster // state listener , but is also activated from this clusterChanged method clusterService.add(this); clusterService.add(nativeUserStore); clusterService.add(nativeRolesStore); - clusterService.add(new SecurityTemplateService(settings, clusterService, client, threadPool)); + clusterService.add(new SecurityTemplateService(settings, clusterService, client)); clusterService.addLifecycleListener(new LifecycleListener() { @Override diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityTemplateService.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityTemplateService.java index c4594daaa23..b17fa419913 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityTemplateService.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityTemplateService.java @@ -6,25 +6,34 @@ package org.elasticsearch.xpack.security; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.xpack.template.TemplateUtils; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; +import java.io.IOException; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -35,76 +44,228 @@ public class SecurityTemplateService extends AbstractComponent implements Cluste public static final String SECURITY_INDEX_NAME = ".security"; public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; + private static final String SECURITY_VERSION_STRING = "security-version"; - private final ThreadPool threadPool; private final InternalClient client; - private final AtomicBoolean templateCreationPending = new AtomicBoolean(false); + final AtomicBoolean templateCreationPending = new AtomicBoolean(false); + final AtomicBoolean updateMappingPending = new AtomicBoolean(false); public SecurityTemplateService(Settings settings, ClusterService clusterService, - InternalClient client, ThreadPool threadPool) { + InternalClient client) { super(settings); - this.threadPool = threadPool; this.client = client; clusterService.add(this); } - private void createSecurityTemplate() { - try (InputStream is = getClass().getResourceAsStream("/" + SECURITY_TEMPLATE_NAME + ".json")) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Streams.copy(is, out); - final byte[] template = out.toByteArray(); - logger.debug("putting the security index template"); - PutIndexTemplateRequest putTemplateRequest = client.admin().indices() - .preparePutTemplate(SECURITY_TEMPLATE_NAME).setSource(template).request(); - PutIndexTemplateResponse templateResponse = client.admin().indices().putTemplate(putTemplateRequest).get(); - if (templateResponse.isAcknowledged() == false) { - throw new ElasticsearchException("adding template for security index was not acknowledged"); - } - } catch (Exception e) { - logger.error("failed to create security index template [{}]", - e, SECURITY_INDEX_NAME); - throw new IllegalStateException("failed to create security index template [" + - SECURITY_INDEX_NAME + "]", e); - } - } - @Override public void clusterChanged(ClusterChangedEvent event) { - if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + if (event.localNodeMaster() == false) { + return; + } + ClusterState state = event.state(); + if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { // wait until the gateway has recovered from disk, otherwise we think may not have .security-audit- // but they may not have been restored from the cluster state on disk logger.debug("template service waiting until state has been recovered"); return; } - - IndexRoutingTable securityIndexRouting = event.state().routingTable().index(SECURITY_INDEX_NAME); - - if (securityIndexRouting == null) { - if (event.localNodeMaster()) { - ClusterState state = event.state(); - // norelease we need to add some checking in the event the template needs to be updated and also the mappings need to be - // updated on index too! - IndexTemplateMetaData templateMeta = state.metaData().templates().get(SECURITY_TEMPLATE_NAME); - final boolean createTemplate = (templateMeta == null); - - if (createTemplate && templateCreationPending.compareAndSet(false, true)) { - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - logger.warn("failed to create security index template", e); - templateCreationPending.set(false); - } - - @Override - protected void doRun() throws Exception { - if (createTemplate) { - createSecurityTemplate(); - } - templateCreationPending.set(false); - } - }); - } + if (securityTemplateExistsAndIsUpToDate(state, logger) == false) { + updateSecurityTemplate(); + } + // make sure mapping is up to date + if (state.metaData().getIndices() != null) { + if (securityIndexMappingUpToDate(state, logger) == false) { + updateSecurityMapping(); } } } + + private void updateSecurityTemplate() { + // only put the template if this is not already in progress + if (templateCreationPending.compareAndSet(false, true)) { + putSecurityTemplate(); + } + } + + private void updateSecurityMapping() { + // only update the mapping if this is not already in progress + if (updateMappingPending.compareAndSet(false, true) ) { + putSecurityMappings(); + } + } + + private void putSecurityMappings() { + BytesReference template; + try { + template = TemplateUtils.load("/" + SECURITY_TEMPLATE_NAME + ".json"); + } catch (IOException e) { + updateMappingPending.set(false); + logger.error("failed to load security index template", e); + throw new ElasticsearchException("failed to load security index template", e); + } + + Map typeMappingMap; + try { + XContentParser xParser = XContentFactory.xContent(template).createParser(template); + typeMappingMap = xParser.map(); + } catch (IOException e) { + updateMappingPending.set(false); + logger.error("failed to parse the security index template", e); + throw new ElasticsearchException("failed to parse the security index template", e); + } + + // here go over all types found in the template and update them + // we need to wait for all types + final Map updateResults = ConcurrentCollections.newConcurrentMap(); + Map typeMappings = (Map) typeMappingMap.get("mappings"); + int expectedResults = typeMappings.size(); + for (String type : typeMappings.keySet()) { + // get the mappings from the template definition + Map typeMapping = (Map) typeMappings.get(type); + // update the mapping + putSecurityMapping(updateResults, expectedResults, type, typeMapping); + } + } + + private void putSecurityMapping(final Map updateResults, int expectedResults, + final String type, Map typeMapping) { + logger.debug("updating mapping of the security index for type [{}]", type); + PutMappingRequest putMappingRequest = client.admin().indices() + .preparePutMapping(SECURITY_INDEX_NAME).setSource(typeMapping).setType(type).request(); + client.admin().indices().putMapping(putMappingRequest, new ActionListener() { + @Override + public void onResponse(PutMappingResponse putMappingResponse) { + if (putMappingResponse.isAcknowledged() == false) { + updateMappingPending.set(false); + throw new ElasticsearchException("update mapping for [{}] security index " + + "was not acknowledged", type); + } else { + updateResults.put(type, putMappingResponse); + if (updateResults.size() == expectedResults) { + updateMappingPending.set(false); + } + } + } + + @Override + public void onFailure(Exception e) { + updateMappingPending.set(false); + logger.warn("failed to update mapping for [{}] on security index", e, type); + } + }); + } + + private void putSecurityTemplate() { + logger.debug("putting the security index template"); + BytesReference template; + try { + template = TemplateUtils.load("/" + SECURITY_TEMPLATE_NAME + ".json"); + } catch (Exception e) { + templateCreationPending.set(false); + logger.error("failed to load security index templates for [{}]", + e, SECURITY_INDEX_NAME); + throw new ElasticsearchException("failed to load security index template", e); + } + + PutIndexTemplateRequest putTemplateRequest = client.admin().indices() + .preparePutTemplate(SECURITY_TEMPLATE_NAME).setSource(template).request(); + client.admin().indices().putTemplate(putTemplateRequest, new ActionListener() { + @Override + public void onResponse(PutIndexTemplateResponse putIndexTemplateResponse) { + templateCreationPending.set(false); + if (putIndexTemplateResponse.isAcknowledged() == false) { + throw new ElasticsearchException("put template for security index was not acknowledged"); + } + } + + @Override + public void onFailure(Exception e) { + templateCreationPending.set(false); + logger.warn("failed to put security index template", e); + } + }); + } + + static boolean securityIndexMappingUpToDate(ClusterState clusterState, ESLogger logger) { + IndexMetaData indexMetaData = clusterState.metaData().getIndices().get(SECURITY_INDEX_NAME); + if (indexMetaData != null) { + for (Object object : indexMetaData.getMappings().values().toArray()) { + MappingMetaData mappingMetaData = (MappingMetaData) object; + if (mappingMetaData.type().equals(MapperService.DEFAULT_MAPPING)) { + continue; + } + try { + if (containsCorrectVersion(mappingMetaData.sourceAsMap()) == false) { + return false; + } + } catch (IOException e) { + logger.error("Cannot parse the mapping for security index.", e); + throw new ElasticsearchException("Cannot parse the mapping for security index.", e); + } + } + return true; + } else { + // index does not exist so when we create it it will be up to date + return true; + } + } + + static boolean securityTemplateExistsAndIsUpToDate(ClusterState state, ESLogger logger) { + IndexTemplateMetaData templateMeta = state.metaData().templates().get(SECURITY_TEMPLATE_NAME); + if (templateMeta == null) { + return false; + } + ImmutableOpenMap mappings = templateMeta.getMappings(); + // check all mappings contain correct version in _meta + // we have to parse the source here which is annoying + for (Object typeMapping : mappings.values().toArray()) { + CompressedXContent typeMappingXContent = (CompressedXContent) typeMapping; + try (XContentParser xParser = XContentFactory.xContent(typeMappingXContent.toString()) + .createParser(typeMappingXContent.toString())) { + Map typeMappingMap = xParser.map(); + // should always contain one entry with key = typename + assert (typeMappingMap.size() == 1); + String key = typeMappingMap.keySet().iterator().next(); + // get the actual mapping entries + Map mappingMap = (Map) typeMappingMap.get(key); + if (containsCorrectVersion(mappingMap) == false) { + return false; + } + } catch (IOException e) { + logger.error("Cannot parse the template for security index.", e); + throw new IllegalStateException("Cannot parse the template for security index.", e); + } + } + return true; + } + + private static boolean containsCorrectVersion(Map typeMappingMap) { + if (typeMappingMap.get("_meta") == null) { + // pre 5.0, cannot be up to date + return false; + } + if (((Map) typeMappingMap.get("_meta")).get(SECURITY_VERSION_STRING) == null) { + // pre 5.0, cannot be up to date + return false; + } + if (((Map) typeMappingMap.get("_meta")).get(SECURITY_VERSION_STRING) + .equals(Version.CURRENT.toString()) == false) { + // wrong version + return false; + } + return true; + } + + public static boolean securityIndexMappingAndTemplateUpToDate(ClusterState clusterState, ESLogger logger) { + if (SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterState, logger) == false) { + logger.debug("security template [{}] does not exist or is not up to date, so service cannot start", + SecurityTemplateService.SECURITY_TEMPLATE_NAME); + return false; + } + if (SecurityTemplateService.securityIndexMappingUpToDate(clusterState, logger) == false) { + logger.debug("mapping for security index not up to date, so service cannot start"); + return false; + } + return true; + } } diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index c591ccf1e9f..4a43543272f 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -77,6 +77,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.security.Security.setting; +import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateUpToDate; /** * ESNativeUsersStore is a {@code UserStore} that, instead of reading from a @@ -501,9 +502,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL return false; } - if (clusterState.metaData().templates().get(SecurityTemplateService.SECURITY_TEMPLATE_NAME) == null) { - logger.debug("native users template [{}] does not exist, so service cannot start", - SecurityTemplateService.SECURITY_TEMPLATE_NAME); + if (securityIndexMappingAndTemplateUpToDate(clusterState, logger) == false) { return false; } diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index f2e2b005c9f..df0a8d028ef 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -73,6 +73,7 @@ import org.elasticsearch.threadpool.ThreadPool.Cancellable; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.security.Security.setting; +import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateUpToDate; /** * ESNativeRolesStore is a {@code RolesStore} that, instead of reading from a @@ -134,14 +135,7 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C logger.debug("native roles store waiting until gateway has recovered from disk"); return false; } - - if (clusterState.metaData().templates().get(SecurityTemplateService.SECURITY_TEMPLATE_NAME) == null) { - logger.debug("native roles template [{}] does not exist, so service cannot start", - SecurityTemplateService.SECURITY_TEMPLATE_NAME); - return false; - } - // Okay to start... - return true; + return securityIndexMappingAndTemplateUpToDate(clusterState, logger); } public void start() { diff --git a/elasticsearch/x-pack/security/src/main/resources/security-index-template.json b/elasticsearch/x-pack/security/src/main/resources/security-index-template.json index d972f0e2082..5333dcbca16 100644 --- a/elasticsearch/x-pack/security/src/main/resources/security-index-template.json +++ b/elasticsearch/x-pack/security/src/main/resources/security-index-template.json @@ -32,6 +32,9 @@ }, "mappings" : { "user" : { + "_meta": { + "security-version": "5.0.0-alpha5" + }, "dynamic" : "strict", "properties" : { "username" : { @@ -59,6 +62,9 @@ } }, "role" : { + "_meta": { + "security-version": "5.0.0-alpha5" + }, "dynamic" : "strict", "properties" : { "cluster" : { @@ -94,6 +100,9 @@ } }, "reserved-user" : { + "_meta": { + "security-version": "5.0.0-alpha5" + }, "dynamic" : "strict", "properties" : { "password": { diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityTemplateServiceTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityTemplateServiceTests.java new file mode 100644 index 00000000000..ece5a2879c8 --- /dev/null +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityTemplateServiceTests.java @@ -0,0 +1,342 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.LocalTransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.xpack.template.TemplateUtils; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.elasticsearch.xpack.security.SecurityTemplateService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.SecurityTemplateService.SECURITY_TEMPLATE_NAME; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SecurityTemplateServiceTests extends ESTestCase { + private InternalClient client; + private TransportClient transportClient; + private ThreadPool threadPool; + private ClusterService clusterService; + SecurityTemplateService securityTemplateService; + private static final ClusterState EMPTY_CLUSTER_STATE = + new ClusterState.Builder(new ClusterName("test-cluster")).build(); + + CopyOnWriteArrayList listeners; + + @Before + public void setup() { + DiscoveryNode localNode = mock(DiscoveryNode.class); + when(localNode.getHostAddress()).thenReturn(LocalTransportAddress.buildUnique().toString()); + clusterService = mock(ClusterService.class); + when(clusterService.localNode()).thenReturn(localNode); + + threadPool = new TestThreadPool("security template service tests"); + transportClient = new MockTransportClient(Settings.EMPTY); + class IClient extends InternalClient { + IClient(Client transportClient) { + super(Settings.EMPTY, null, transportClient, null); + } + + @Override + protected , Response extends ActionResponse, RequestBuilder extends + ActionRequestBuilder> void doExecute( + Action action, Request request + , ActionListener listener) { + listeners.add(listener); + } + } + client = new IClient(transportClient); + securityTemplateService = new SecurityTemplateService(Settings.EMPTY, clusterService, client); + listeners = new CopyOnWriteArrayList<>(); + } + + @After + public void stop() throws InterruptedException { + if (transportClient != null) { + transportClient.close(); + } + terminate(threadPool); + } + + public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { + String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); + assertTrue(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger)); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(0)); + } + + public void testFaultyIndexTemplateIsIdentifiedAsNotUpToDate() throws IOException { + String templateString = "/wrong-version-" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); + assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger)); + checkTemplateUpdateWorkCorrectly(clusterStateBuilder); + } + + private void checkTemplateUpdateWorkCorrectly(ClusterState.Builder clusterStateBuilder) throws IOException { + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(1)); + assertTrue(securityTemplateService.templateCreationPending.get()); + + // if we do it again this should not send an update + ActionListener listener = listeners.get(0); + listeners.clear(); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(0)); + assertTrue(securityTemplateService.templateCreationPending.get()); + + // if we now simulate an error... + listener.onFailure(new Exception()); + assertFalse(securityTemplateService.templateCreationPending.get()); + + // ... we should be able to send a new update + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(1)); + assertTrue(securityTemplateService.templateCreationPending.get()); + + // now check what happens if we get back an unacknowledged response + try { + listeners.get(0).onResponse(new TestPutIndexTemplateResponse()); + fail("this hould have failed because request was not acknowledged"); + } catch (ElasticsearchException e) { + } + assertFalse(securityTemplateService.updateMappingPending.get()); + + // and now let's see what happens if we get back a response + listeners.clear(); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertTrue(securityTemplateService.templateCreationPending.get()); + assertThat(listeners.size(), equalTo(1)); + listeners.get(0).onResponse(new TestPutIndexTemplateResponse(true)); + assertFalse(securityTemplateService.templateCreationPending.get()); + } + + public void testMissingIndexTemplateIsIdentifiedAsMissing() throws IOException { + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state()); + // add the correct mapping + String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexMetaData.Builder indexMeta = createIndexMetadata(mappingString); + MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); + builder.put(indexMeta); + clusterStateBuilder.metaData(builder); + assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger)); + checkTemplateUpdateWorkCorrectly(clusterStateBuilder); + } + + public void testMissingVersionIndexTemplateIsIdentifiedAsNotUpToDate() throws IOException { + String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); + assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger)); + checkTemplateUpdateWorkCorrectly(clusterStateBuilder); + } + + public void testOutdatedMappingIsIdentifiedAsNotUpToDate() throws IOException { + String templateString = "/wrong-version-" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString); + assertFalse(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger)); + checkMappingUpdateWorkCorrectly(clusterStateBuilder); + } + + private void checkMappingUpdateWorkCorrectly(ClusterState.Builder clusterStateBuilder) { + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(3)); // we have three types in the mapping + assertTrue(securityTemplateService.updateMappingPending.get()); + + // if we do it again this should not send an update + ActionListener listener = listeners.get(0); + listeners.clear(); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(0)); + assertTrue(securityTemplateService.updateMappingPending.get()); + + // if we now simulate an error... + listener.onFailure(new Exception()); + assertFalse(securityTemplateService.updateMappingPending.get()); + + // ... we should be able to send a new update + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(3)); + assertTrue(securityTemplateService.updateMappingPending.get()); + + // now check what happens if we get back an unacknowledged response + try { + listeners.get(0).onResponse(new TestPutMappingResponse()); + fail("this hould have failed because request was not acknowledged"); + } catch (ElasticsearchException e) { + } + assertFalse(securityTemplateService.updateMappingPending.get()); + + // and now check what happens if we get back an acknowledged response + listeners.clear(); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(3)); // we have three types in the mapping + int counter = 0; + for (ActionListener actionListener : listeners) { + actionListener.onResponse(new TestPutMappingResponse(true)); + if (counter++ < 2) { + assertTrue(securityTemplateService.updateMappingPending.get()); + } else { + assertFalse(securityTemplateService.updateMappingPending.get()); + } + } + } + + public void testUpToDateMappingIsIdentifiedAstUpToDate() throws IOException { + String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString); + assertTrue(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger)); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(0)); + } + + public void testMissingVersionMappingIsIdentifiedAsNotUpToDate() throws IOException { + String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString); + assertFalse(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger)); + checkMappingUpdateWorkCorrectly(clusterStateBuilder); + } + + public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { + ClusterState.Builder clusterStateBuilder = ClusterState.builder(new ClusterName("test-cluster")); + String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(mappingString); + MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); + builder.put(templateMeta); + clusterStateBuilder.metaData(builder); + assertTrue(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger)); + securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(listeners.size(), equalTo(0)); + } + + private ClusterState.Builder createClusterStateWithMapping(String templateString) throws IOException { + IndexMetaData.Builder indexMetaData = createIndexMetadata(templateString); + ImmutableOpenMap.Builder mapBuilder = ImmutableOpenMap.builder(); + mapBuilder.put(SECURITY_INDEX_NAME, indexMetaData.build()); + MetaData.Builder metaDataBuidler = new MetaData.Builder(); + metaDataBuidler.indices(mapBuilder.build()); + String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(mappingString); + metaDataBuidler.put(templateMeta); + ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); + clusterStateBuilder.metaData(metaDataBuidler.build()); + return clusterStateBuilder; + } + + private IndexMetaData.Builder createIndexMetadata(String templateString) throws IOException { + BytesReference template = TemplateUtils.load(templateString); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source(template); + IndexMetaData.Builder indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME); + indexMetaData.settings(Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build()); + + for (Map.Entry entry : request.mappings().entrySet()) { + indexMetaData.putMapping(entry.getKey(), entry.getValue()); + } + return indexMetaData; + } + + private ClusterState.Builder createClusterStateWithTemplate(String templateString) throws IOException { + IndexTemplateMetaData.Builder templateBuilder = getIndexTemplateMetaData(templateString); + MetaData.Builder metaDataBuidler = new MetaData.Builder(); + metaDataBuidler.put(templateBuilder); + // add the correct mapping no matter what the template + String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexMetaData.Builder indexMeta = createIndexMetadata(mappingString); + metaDataBuidler.put(indexMeta); + return ClusterState.builder(state()) + .metaData(metaDataBuidler.build()); + } + + private IndexTemplateMetaData.Builder getIndexTemplateMetaData(String templateString) throws IOException { + BytesReference template = TemplateUtils.load(templateString); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source(template); + IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(SECURITY_TEMPLATE_NAME); + for (Map.Entry entry : request.mappings().entrySet()) { + templateBuilder.putMapping(entry.getKey(), entry.getValue()); + } + return templateBuilder; + } + + // cluster state where local node is master + private static ClusterState state() { + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + discoBuilder.masterNodeId("1"); + discoBuilder.localNodeId("1"); + ClusterState.Builder state = ClusterState.builder(new ClusterName("test-cluster")); + state.nodes(discoBuilder); + state.metaData(MetaData.builder().generateClusterUuidIfNeeded()); + return state.build(); + } + + private static class TestPutMappingResponse extends PutMappingResponse { + public TestPutMappingResponse(boolean acknowledged) { + super(acknowledged); + } + + public TestPutMappingResponse() { + super(); + } + } + + private static class TestPutIndexTemplateResponse extends PutIndexTemplateResponse { + public TestPutIndexTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + + public TestPutIndexTemplateResponse() { + super(); + } + } +} diff --git a/elasticsearch/x-pack/security/src/test/resources/missing-version-security-index-template.json b/elasticsearch/x-pack/security/src/test/resources/missing-version-security-index-template.json new file mode 100644 index 00000000000..d972f0e2082 --- /dev/null +++ b/elasticsearch/x-pack/security/src/test/resources/missing-version-security-index-template.json @@ -0,0 +1,107 @@ +{ + "template" : ".security", + "order" : 1000, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "auto_expand_replicas" : "0-all", + "analysis" : { + "filter" : { + "email" : { + "type" : "pattern_capture", + "preserve_original" : 1, + "patterns" : [ + "([^@]+)", + "(\\p{L}+)", + "(\\d+)", + "@(.+)" + ] + } + }, + "analyzer" : { + "email" : { + "tokenizer" : "uax_url_email", + "filter" : [ + "email", + "lowercase", + "unique" + ] + } + } + } + }, + "mappings" : { + "user" : { + "dynamic" : "strict", + "properties" : { + "username" : { + "type" : "keyword" + }, + "roles" : { + "type" : "keyword" + }, + "password" : { + "type" : "keyword", + "index" : false, + "doc_values": false + }, + "full_name" : { + "type" : "text" + }, + "email" : { + "type" : "text", + "analyzer" : "email" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + } + } + }, + "role" : { + "dynamic" : "strict", + "properties" : { + "cluster" : { + "type" : "keyword" + }, + "indices" : { + "type" : "object", + "properties" : { + "fields" : { + "type" : "keyword" + }, + "names" : { + "type" : "keyword" + }, + "privileges" : { + "type" : "keyword" + }, + "query" : { + "type" : "keyword" + } + } + }, + "name" : { + "type" : "keyword" + }, + "run_as" : { + "type" : "keyword" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + } + } + }, + "reserved-user" : { + "dynamic" : "strict", + "properties" : { + "password": { + "type" : "keyword", + "index" : false, + "doc_values" : false + } + } + } + } +} diff --git a/elasticsearch/x-pack/security/src/test/resources/wrong-version-security-index-template.json b/elasticsearch/x-pack/security/src/test/resources/wrong-version-security-index-template.json new file mode 100644 index 00000000000..55197b3b441 --- /dev/null +++ b/elasticsearch/x-pack/security/src/test/resources/wrong-version-security-index-template.json @@ -0,0 +1,116 @@ +{ + "template" : ".security", + "order" : 1000, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "auto_expand_replicas" : "0-all", + "analysis" : { + "filter" : { + "email" : { + "type" : "pattern_capture", + "preserve_original" : 1, + "patterns" : [ + "([^@]+)", + "(\\p{L}+)", + "(\\d+)", + "@(.+)" + ] + } + }, + "analyzer" : { + "email" : { + "tokenizer" : "uax_url_email", + "filter" : [ + "email", + "lowercase", + "unique" + ] + } + } + } + }, + "mappings" : { + "user" : { + "_meta": { + "security-version": "4.0.0-alpha5" + }, + "dynamic" : "strict", + "properties" : { + "username" : { + "type" : "keyword" + }, + "roles" : { + "type" : "keyword" + }, + "password" : { + "type" : "keyword", + "index" : false, + "doc_values": false + }, + "full_name" : { + "type" : "text" + }, + "email" : { + "type" : "text", + "analyzer" : "email" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + } + } + }, + "role" : { + "_meta": { + "security-version": "5.0.0-alpha5" + }, + "dynamic" : "strict", + "properties" : { + "cluster" : { + "type" : "keyword" + }, + "indices" : { + "type" : "object", + "properties" : { + "fields" : { + "type" : "keyword" + }, + "names" : { + "type" : "keyword" + }, + "privileges" : { + "type" : "keyword" + }, + "query" : { + "type" : "keyword" + } + } + }, + "name" : { + "type" : "keyword" + }, + "run_as" : { + "type" : "keyword" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + } + } + }, + "reserved-user" : { + "_meta": { + "security-version": "5.0.0-alpha5" + }, + "dynamic" : "strict", + "properties" : { + "password": { + "type" : "keyword", + "index" : false, + "doc_values" : false + } + } + } + } +}