From b76c85e7fd900c40d973398239fbf0fab7696097 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 19 Oct 2017 13:53:24 +0200 Subject: [PATCH 01/10] Docs: Change port to be an integer in htttp input docs Original commit: elastic/x-pack-elasticsearch@768ec54e035fc84439f37e4b37238ec1e16c7470 --- docs/en/watcher/input/http.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/watcher/input/http.asciidoc b/docs/en/watcher/input/http.asciidoc index b4cdcdbf4b3..2d1f7e2380f 100644 --- a/docs/en/watcher/input/http.asciidoc +++ b/docs/en/watcher/input/http.asciidoc @@ -73,7 +73,7 @@ Stats] API and enables the `human` attribute: "http" : { "request" : { "host" : "host.domain", - "port" : "9200", + "port" : 9200, "path" : "/_cluster/stats", "params" : { "human" : "true" <1> @@ -98,7 +98,7 @@ a username and password to access `myservice`: "http" : { "request" : { "host" : "host.domain", - "port" : "9200", + "port" : 9200, "path" : "/myservice", "auth" : { "basic" : { From 652f6560b81e2d5083eadf6b68f5f3267c70ffe3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 10 Oct 2017 10:00:34 +0200 Subject: [PATCH 02/10] security: Always allow access to a rootdoc's nested documents if access to rootdoc is allowed relates elastic/x-pack-elasticsearch#2665 Original commit: elastic/x-pack-elasticsearch@2bbddd1dd2bd89c74bff08ddbd1bccd948383393 --- .../SecurityIndexSearcherWrapper.java | 23 +++++---- .../DocumentLevelSecurityTests.java | 47 +++++++++++++++++-- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index be42573bd3a..3f47c5a360a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -17,24 +17,21 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.SparseFixedBitSet; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.Action; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.FilterClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; @@ -46,7 +43,6 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; -import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; @@ -139,10 +135,17 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper { QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser); verifyRoleQuery(queryBuilder); failIfQueryUsesClient(queryBuilder, queryShardContext); - ParsedQuery parsedQuery = queryShardContext.toFilter(queryBuilder); - filter.add(parsedQuery.query(), SHOULD); + Query roleQuery = queryShardContext.toFilter(queryBuilder).query(); + filter.add(roleQuery, SHOULD); + if (queryShardContext.getMapperService().hasNested()) { + // If access is allowed on root doc then also access is allowed on all nested docs of that root document: + BitSetProducer rootDocs = queryShardContext.bitsetFilter(Queries.newNonNestedFilter()); + ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs); + filter.add(includeNestedDocs, SHOULD); + } } } + // at least one of the queries should match filter.setMinimumNumberShouldMatch(1); reader = DocumentSubsetReader.wrap(reader, bitsetFilterCache, new ConstantScoreQuery(filter.build())); diff --git a/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 96cd8e250a1..ef307bd9d92 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -23,6 +23,8 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndicesRequestCache; @@ -37,8 +39,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortMode; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.xpack.XPackPlugin; @@ -52,6 +52,7 @@ import java.util.HashMap; import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery; @@ -658,7 +659,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase { } public void testParentChild_joinField() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + XContentBuilder mapping = jsonBuilder().startObject() .startObject("properties") .startObject("join_field") .field("type", "join") @@ -903,4 +904,44 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase { assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field1").toString(), equalTo("value3")); } + public void testNestedInnerHits() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("type1", "field1", "type=text", "nested_field", "type=nested") + ); + client().prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested_field") + .startObject() + .field("field2", "value2") + .endObject() + .endArray() + .endObject()) + .get(); + client().prepareIndex("test", "type1", "2") + .setSource(jsonBuilder().startObject() + .field("field1", "value2") + .startArray("nested_field") + .startObject() + .field("field2", "value2") + .endObject() + .endArray() + .endObject()) + .get(); + refresh("test"); + + SearchResponse response = client() + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) + .prepareSearch("test") + .setQuery(QueryBuilders.nestedQuery("nested_field", QueryBuilders.termQuery("nested_field.field2", "value2"), + ScoreMode.None).innerHit(new InnerHitBuilder())) + .get(); + assertHitCount(response, 1); + assertSearchHits(response, "1"); + assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getId(), equalTo("1")); + assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getNestedIdentity().getOffset(), equalTo(0)); + assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getSourceAsString(), + equalTo("{\"nested_field\":{\"field2\":\"value2\"}}")); + } + } From c9682d02d434795878a4ec2dd8bc787faa5f9c0a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 23 Oct 2017 09:58:33 +0200 Subject: [PATCH 03/10] fix test Original commit: elastic/x-pack-elasticsearch@7ca5e0fb54367eefd170b6b4ba8e9168be049780 --- .../elasticsearch/integration/DocumentLevelSecurityTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index ef307bd9d92..52ca23183b0 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -941,7 +941,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getId(), equalTo("1")); assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getNestedIdentity().getOffset(), equalTo(0)); assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getSourceAsString(), - equalTo("{\"nested_field\":{\"field2\":\"value2\"}}")); + equalTo("{\"field2\":\"value2\"}")); } } From 6f437c973adc8e61fde43ecd1934496d5c68d199 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 23 Oct 2017 11:57:40 +0200 Subject: [PATCH 04/10] Watcher: Ensure all templates exist before starting watcher (elastic/x-pack-elasticsearch#2765) This is to ensure that the required templates exist (which are added by the WatcherIndexTemplateRegistry) before actually starting watcher. Relates elastic/x-pack-elasticsearch#2761 Original commit: elastic/x-pack-elasticsearch@568088061fec776a4a23eac8e338227d9dc63327 --- .../watcher/WatcherLifeCycleService.java | 8 +++ .../support/WatcherIndexTemplateRegistry.java | 6 ++ .../watcher/WatcherLifeCycleServiceTests.java | 61 ++++++++++++++++++- .../WatcherIndexTemplateRegistryTests.java | 24 +++++++- 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java index 50aff73559e..5941e2a32ef 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java @@ -21,6 +21,7 @@ import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.upgrade.Upgrade; import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; +import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.xpack.watcher.watch.Watch; import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils; @@ -76,6 +77,13 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste return; } + // ensure that templates are existing before starting watcher + // the watcher index template registry is independent from watcher being started or stopped + if (WatcherIndexTemplateRegistry.validate(state) == false) { + logger.debug("not starting watcher, watcher templates are missing in the cluster state"); + return; + } + if (watcherService.validate(state)) { logger.trace("starting... (based on cluster state version [{}]) (manual [{}])", state.getVersion(), manual); try { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index 084d66096a4..1d69781cdad 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -127,6 +127,12 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C }); } + public static boolean validate(ClusterState state) { + return state.getMetaData().getTemplates().containsKey(HISTORY_TEMPLATE_NAME) && + state.getMetaData().getTemplates().containsKey(TRIGGERED_TEMPLATE_NAME) && + state.getMetaData().getTemplates().containsKey(WATCHES_TEMPLATE_NAME); + } + public static class TemplateConfig { private final String templateName; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java index 6a63303d541..560cc4c312d 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -30,6 +31,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; +import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry; import org.elasticsearch.xpack.watcher.watch.Watch; import org.junit.Before; import org.mockito.stubbing.Answer; @@ -82,6 +84,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(new Index(Watch.INDEX, "foo")).build(); ClusterState clusterState = ClusterState.builder(new ClusterName("my-cluster")) + .metaData(MetaData.builder() + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)) + .build()) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) .routingTable(RoutingTable.builder().add(watchRoutingTable).build()) .build(); @@ -114,6 +121,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { ClusterState clusterState = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) .routingTable(RoutingTable.builder().add(watchRoutingTable).build()) + .metaData(MetaData.builder() + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)) + .build()) .build(); when(watcherService.validate(clusterState)).thenReturn(true); @@ -138,6 +150,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { // no change, keep going clusterState = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) + .metaData(MetaData.builder() + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)) + .build()) .build(); when(watcherService.state()).thenReturn(WatcherState.STARTED); lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterState, clusterState)); @@ -146,6 +163,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { ClusterState previousClusterState = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) + .metaData(MetaData.builder() + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)) + .build()) .build(); when(watcherService.validate(clusterState)).thenReturn(true); when(watcherService.state()).thenReturn(WatcherState.STOPPED); @@ -413,7 +435,14 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { .add(newNode("oldNode", VersionUtils.randomVersionBetween(random(), Version.V_5_5_0, Version.V_6_0_0_alpha2))) .build(); - ClusterState state = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build(); + ClusterState state = ClusterState.builder(new ClusterName("my-cluster")) + .nodes(nodes) + .metaData(MetaData.builder() + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)) + .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)) + .build()) + .build(); when(watcherService.validate(eq(state))).thenReturn(true); when(watcherService.state()).thenReturn(WatcherState.STOPPED); @@ -421,6 +450,36 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { verify(watcherService).start(any(ClusterState.class)); } + public void testWatcherServiceDoesNotStartIfIndexTemplatesAreMissing() throws Exception { + DiscoveryNodes nodes = new DiscoveryNodes.Builder() + .masterNodeId("node_1").localNodeId("node_1") + .add(newNode("node_1")) + .build(); + + MetaData.Builder metaDataBuilder = MetaData.builder(); + boolean isHistoryTemplateAdded = randomBoolean(); + if (isHistoryTemplateAdded) { + metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME)); + } + boolean isTriggeredTemplateAdded = randomBoolean(); + if (isTriggeredTemplateAdded) { + metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME)); + } + boolean isWatchesTemplateAdded = randomBoolean(); + if (isWatchesTemplateAdded) { + // ensure not all templates are added, otherwise life cycle service would start + if ((isHistoryTemplateAdded || isTriggeredTemplateAdded) == false) { + metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME)); + } + } + ClusterState state = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).metaData(metaDataBuilder).build(); + when(watcherService.validate(eq(state))).thenReturn(true); + when(watcherService.state()).thenReturn(WatcherState.STOPPED); + + lifeCycleService.clusterChanged(new ClusterChangedEvent("any", state, state)); + verify(watcherService, times(0)).start(any(ClusterState.class)); + } + private static DiscoveryNode newNode(String nodeName) { return newNode(nodeName, Version.CURRENT); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index 1465042e1ec..07404ec8c1b 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -13,13 +13,13 @@ import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -36,6 +36,7 @@ import java.util.List; import static org.elasticsearch.mock.orig.Mockito.verify; import static org.elasticsearch.mock.orig.Mockito.when; +import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; @@ -51,8 +52,6 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { @Before public void createRegistryAndClient() { - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.emptySet()); - ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); when(threadPool.generic()).thenReturn(EsExecutors.newDirectExecutorService()); @@ -108,6 +107,25 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { verify(client, times(4)).execute(anyObject(), argumentCaptor.capture(), anyObject()); } + public void testThatTemplatesExist() { + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history")), is(false)); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history", ".triggered_watches", ".watches")), + is(false)); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME, + ".triggered_watches", ".watches")), is(true)); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME, + ".triggered_watches", ".watches", "whatever", "else")), is(true)); + } + + private ClusterState createClusterState(String ... existingTemplates) { + MetaData.Builder metaDataBuilder = MetaData.builder(); + for (String templateName : existingTemplates) { + metaDataBuilder.put(IndexTemplateMetaData.builder(templateName)); + } + + return ClusterState.builder(new ClusterName("foo")).metaData(metaDataBuilder.build()).build(); + } + private static class TestPutIndexTemplateResponse extends PutIndexTemplateResponse { TestPutIndexTemplateResponse(boolean acknowledged) { super(acknowledged); From 403912b8a278a32648f25ef82354746ede0ec37e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 24 Oct 2017 13:50:35 +0300 Subject: [PATCH 05/10] SecureSettings ignored by customAuditIndexSettings (elastic/x-pack-elasticsearch#2748) customAuditIndexSettings does not submit SecureSettings with putIndexMapping. relates elastic/x-pack-elasticsearch#2705 * Randomize SecureSetting in testcase * Apply feedback Original commit: elastic/x-pack-elasticsearch@1a5414b057657e69208f29f9d29a5d580a14b6c5 --- .../security/audit/index/IndexAuditTrail.java | 2 +- .../audit/index/IndexAuditTrailTests.java | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index 605ff27e497..3a437341f4c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -827,7 +827,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl Settings customAuditIndexSettings(Settings nodeSettings) { Settings newSettings = Settings.builder() - .put(INDEX_SETTINGS.get(nodeSettings)) + .put(INDEX_SETTINGS.get(nodeSettings), false) .build(); if (newSettings.names().isEmpty()) { return Settings.EMPTY; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index 1b3a9cfeb2a..637ef308958 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -22,6 +22,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.KeyStoreWrapper; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.plugins.Plugin; @@ -275,11 +277,11 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase { return remoteIndexing ? remoteCluster.client() : client(); } - private void initialize() throws IOException, InterruptedException { + private void initialize() throws Exception { initialize(null, null); } - private void initialize(String[] includes, String[] excludes) throws IOException, InterruptedException { + private void initialize(String[] includes, String[] excludes) throws Exception { rollover = randomFrom(HOURLY, DAILY, WEEKLY, MONTHLY); numReplicas = numberOfReplicas(); numShards = numberOfShards(); @@ -288,8 +290,18 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase { if (remoteIndexing) { builder.put(remoteSettings); } + builder.put(settings(rollover, includes, excludes)); + // IndexAuditTrail should ignore secure settings + // they are merged on the master node creating the audit index + if (randomBoolean()) { + MockSecureSettings ignored = new MockSecureSettings(); + if (randomBoolean()) { + ignored.setString(KeyStoreWrapper.SEED_SETTING.getKey(), "non-empty-secure-settings"); + } + builder.setSecureSettings(ignored); + } + Settings settings = builder.build(); - Settings settings = builder.put(settings(rollover, includes, excludes)).build(); logger.info("--> settings: [{}]", settings); DiscoveryNode localNode = mock(DiscoveryNode.class); when(localNode.getHostAddress()).thenReturn(remoteAddress.getAddress()); From 3a6870f0dc3ae5785e0e4014073ba764dee32ab7 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Tue, 24 Oct 2017 16:29:19 +0100 Subject: [PATCH 06/10] [DOCS] Added SSL settings to X-Pack installation (elastic/x-pack-elasticsearch#2733) Original commit: elastic/x-pack-elasticsearch@c88cce763a5ca71491cf7a554dc949f6b0f3f81e --- docs/en/installing-xes.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/installing-xes.asciidoc b/docs/en/installing-xes.asciidoc index 5436ac01b87..643c98454cb 100644 --- a/docs/en/installing-xes.asciidoc +++ b/docs/en/installing-xes.asciidoc @@ -180,12 +180,16 @@ directory. For example, xpack.ssl.key: certs/${node.name}/${node.name}.key <1> xpack.ssl.certificate: certs/${node.name}/${node.name}.crt <2> xpack.ssl.certificate_authorities: certs/ca/ca.crt <3> +xpack.security.transport.ssl.enabled: true +xpack.security.http.ssl.enabled: true <4> ----------------------------------------------------------- <1> If this path does not exist on every node or the file name does not match the `node.name` configuration setting, you must specify the full path to the node key file. <2> Alternatively, specify the full path to the node certificate. <3> Alternatively specify the full path to the CA certificate. +<4> This setting is optional. It enables SSL on the HTTP layer to ensure that +communication between HTTP clients and the cluster is encrypted. -- .. Start {es}. From 73e819b0bdaa1526470e69d67efa225eca42fd5d Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Tue, 24 Oct 2017 16:38:37 +0100 Subject: [PATCH 07/10] [DOCS] Add secure versions of SSL passphrases (elastic/x-pack-elasticsearch#2478) * [DOCS] Add secure versions of SSL passphrases * [DOCS] Add secure xpack settings * [DOCS] Clarify type of keystore * [DOCS] Added secure settings to security page * [DOCS] Clarify X-Pack secure settings * [DOCS] Reformat secure X-Pack settings Original commit: elastic/x-pack-elasticsearch@efe043fb675c00e5c9f8399edf59c2c71dd871e3 --- docs/en/settings/security-settings.asciidoc | 112 +++++++++++++++----- docs/en/settings/ssl-settings.asciidoc | 28 ++++- 2 files changed, 109 insertions(+), 31 deletions(-) diff --git a/docs/en/settings/security-settings.asciidoc b/docs/en/settings/security-settings.asciidoc index 790289e3967..53581a74905 100644 --- a/docs/en/settings/security-settings.asciidoc +++ b/docs/en/settings/security-settings.asciidoc @@ -12,6 +12,11 @@ and perform message authentication, level security>>, <>, and <>. +All of these settings can be added to the `elasticsearch.yml` configuration file, +with the exception of the secure settings, which you add to the {es} keystore. +For more information about creating and updating the {es} keystore, see +<>. + [float] [[general-security-settings]] ==== General Security Settings @@ -24,6 +29,7 @@ need to disable {security} in those `kibana.yml` files. For more information about disabling {security} in specific {kib} instances, see {kibana-ref}/security-settings-kb.html[{kib} Security Settings]. + [float] [[password-security-settings]] ==== Default Password Security Settings @@ -289,9 +295,12 @@ Defaults to `5s` (5 seconds ). Path to a PEM encoded file containing the private key. `ssl.key_passphrase`:: -The passphrase that will be used to decrypt the private key. This value is +The passphrase that is used to decrypt the private key. This value is optional as the key may not be encrypted. +`ssl.secure_key_passphrase` (<>):: +The passphrase that is used to decrypt the private key. + `ssl.certificate`:: Path to a PEM encoded file containing the certificate (or certificate chain) that will be presented to clients when they connect. @@ -310,20 +319,29 @@ Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`. `ssl.keystore.password`:: The password to the keystore. +`ssl.keystore.secure_password` (<>):: +The password to the keystore. + `ssl.keystore.key_password`:: The password for the key in the keystore. Defaults to the keystore password. +`ssl.keystore.secure_key_password`:: +The password for the key in the keystore. Defaults to the keystore password. + `ssl.truststore.path`:: The path to the Java Keystore file that contains the certificates to trust. `ssl.certificate_authorities` and `ssl.truststore.path` may not be used at the same time. +`ssl.truststore.password`:: +The password to the truststore. + +`ssl.truststore.secure_password` (<>):: +The password to the truststore. + `ssl.truststore.type`:: The format of the keystore file. Should be either `jks` to use the Java Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`. -`ssl.truststore.password`:: -The password to the truststore. - `ssl.verification_mode`:: Indicates the type of verification when using `ldaps` to protect against man in the middle attacks and certificate forgery. Values are `none`, `certificate`, @@ -478,13 +496,6 @@ The LDAP Server enforced timeout period for an LDAP search. An `s` at the end indicates seconds, or `ms` indicates milliseconds. Defaults to `5s` (5 seconds ). -`ssl.key`:: -Path to the PEM encoded file containing the private key. - -`ssl.key_passphrase`:: -The passphrase that will be used to decrypt the private key. This value is -optional as the key may not be encrypted. - `ssl.certificate`:: Path to a PEM encoded file containing the certificate (or certificate chain) that will be presented to clients when they connect. @@ -492,6 +503,29 @@ that will be presented to clients when they connect. `ssl.certificate_authorities`:: List of paths to PEM encoded certificate files that should be trusted. +`ssl.key`:: +Path to the PEM encoded file containing the private key. + +`ssl.key_passphrase`:: +The passphrase that is used to decrypt the private key. This value is +optional as the key might not be encrypted. + +`ssl.secure_key_passphrase` (<>):: +The passphrase that is used to decrypt the private key. This value is +optional as the key might not be encrypted. + +`ssl.keystore.key_password`:: +The password for the key in the keystore. Defaults to the keystore password. + +`ssl.keystore.secure_key_password` (<>):: +The password for the key in the keystore. Defaults to the keystore password. + +`ssl.keystore.password`:: +The password to the keystore. + +`ssl.secure_keystore.password` (<>):: +The password to the keystore. + `ssl.keystore.path`:: The path to the Java Keystore file that contains a private key and certificate. @@ -499,11 +533,11 @@ The path to the Java Keystore file that contains a private key and certificate. The format of the keystore file. Should be either `jks` to use the Java Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`. -`ssl.keystore.password`:: -The password to the keystore. +`ssl.truststore.password`:: +The password to the truststore. -`ssl.keystore.key_password`:: -The password for the key in the keystore. Defaults to the keystore password. +`ssl.truststore.secure_password` (<>):: +The password to the truststore. `ssl.truststore.path`:: The path to the Java Keystore file that contains the certificates to trust. @@ -512,9 +546,6 @@ The path to the Java Keystore file that contains the certificates to trust. The format of the truststore file. Should be either `jks` to use the Java Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`. -`ssl.truststore.password`:: -The password to the truststore. - `ssl.verification_mode`:: Indicates the type of verification when using `ldaps` to protect against man in the middle attacks and certificate forgery. Values are `none`, `certificate`, @@ -524,7 +555,7 @@ and `full`. Defaults to the value of `xpack.ssl.verification_mode`. Supported protocols with versions. Defaults to the value of `xpack.ssl.supported_protocols`. -`ssl.cipher_suites` +`ssl.cipher_suites`:: Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[ Java Cryptography Architecture documentation]. Defaults to the value of `xpack.ssl.cipher_suites`. @@ -558,16 +589,19 @@ user's certificate as trusted. Defaults to the trusted certificates configured f See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information. This setting may not be used with `truststore.path`. -`truststore.path`:: -The path of a truststore to use. Defaults to the trusted certificates configured for SSL. -See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information. -This setting may not be used with `certificate_authorities`. +`truststore.algorithm`:: +Algorithm for the truststore. Defaults to `SunX509`. `truststore.password`:: The password for the truststore. Must be provided if `truststore.path` is set. -`truststore.algorithm`:: -Algorithm for the truststore. Defaults to `SunX509`. +`truststore.secure_password` (<>):: +The password for the truststore. + +`truststore.path`:: +The path of a truststore to use. Defaults to the trusted certificates configured for SSL. +See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information. +This setting may not be used with `certificate_authorities`. `files.role_mapping`:: Specifies the {xpack-ref}/security-files.html[location] of the @@ -599,7 +633,7 @@ Defaults to `required`. Controls the verification of certificates. Valid values are `none`, `certificate`, and `full`. Defaults to `full`. -`xpack.ssl.cipher_suites` +`xpack.ssl.cipher_suites`:: Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[ Java Cryptography Architecture documentation]. Defaults to `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256`, `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256`, `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA`, `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA`, @@ -628,8 +662,12 @@ When using PEM encoded files, use the following settings: Path to the PEM encoded file containing the private key. `xpack.ssl.key_passphrase`:: -The passphrase that will be used to decrypt the private key. This value is -optional as the key may not be encrypted. +The passphrase that is used to decrypt the private key. This value is +optional as the key might not be encrypted. + +`xpack.ssl.secure_key_passphrase` ({<>):: +The passphrase that is used to decrypt the private key. This value is +optional as the key might not be encrypted. `xpack.ssl.certificate`:: Path to a PEM encoded file containing the certificate (or certificate chain) @@ -650,16 +688,25 @@ Path to the keystore that holds the private key and certificate. `xpack.ssl.keystore.password`:: Password to the keystore. +`xpack.ssl.keystore.secure_password` (<>):: +Password to the keystore. + `xpack.ssl.keystore.key_password`:: Password for the private key in the keystore. Defaults to the same value as `xpack.ssl.keystore.password`. +`xpack.ssl.keystore.secure_key_password` (<>):: +Password for the private key in the keystore. + `xpack.ssl.truststore.path`:: Path to the truststore file. `xpack.ssl.truststore.password`:: Password to the truststore. +`xpack.ssl.truststore.secure_password` (<>):: +Password to the truststore. + [float] ===== PKCS#12 Files @@ -676,10 +723,16 @@ Set this to `PKCS12`. `xpack.ssl.keystore.password`:: Password to the PKCS#12 file. +`xpack.ssl.keystore.secure_password` (<>):: +Password to the PKCS#12 file. + `xpack.ssl.keystore.key_password`:: Password for the private key in the PKCS12 file. Defaults to the same value as `xpack.ssl.keystore.password`. +`xpack.ssl.keystore.secure_key_password` (<>):: +Password for the private key in the PKCS12 file. + `xpack.ssl.truststore.path`:: Path to the truststore file. @@ -689,6 +742,9 @@ Set this to `PKCS12`. `xpack.ssl.truststore.password`:: Password to the truststore. +`xpack.ssl.truststore.secure_password` (<>):: +Password to the truststore. + [[http-tls-ssl-settings]] :ssl-prefix: xpack.security.http :component: HTTP diff --git a/docs/en/settings/ssl-settings.asciidoc b/docs/en/settings/ssl-settings.asciidoc index 3a4f42eca55..11779ccc798 100644 --- a/docs/en/settings/ssl-settings.asciidoc +++ b/docs/en/settings/ssl-settings.asciidoc @@ -63,8 +63,12 @@ When using PEM encoded files, use the following settings: Path to a PEM encoded file containing the private key. +{ssl-prefix}.ssl.key_passphrase+:: -The passphrase that will be used to decrypt the private key. This value is -optional as the key may not be encrypted. +The passphrase that is used to decrypt the private key. This value is optional +as the key might not be encrypted. + ++{ssl-prefix}.ssl.secure_key_passphrase+ (<>):: +The passphrase that is used to decrypt the private key. This value is optional +as the key might not be encrypted. +{ssl-prefix}.ssl.certificate+:: Path to a PEM encoded file containing the certificate (or certificate chain) @@ -84,16 +88,25 @@ Path to the keystore that holds the private key and certificate. +{ssl-prefix}.ssl.keystore.password+:: Password to the keystore. ++{ssl-prefix}.ssl.keystore.secure_password` (<>):: +Password to the keystore. + +{ssl-prefix}.ssl.keystore.key_password+:: Password for the private key in the keystore. Defaults to the same value as +{ssl-prefix}.ssl.keystore.password+. ++{ssl-prefix}.ssl.keystore.secure_key_password+ (<>):: +Password for the private key in the keystore. + +{ssl-prefix}.ssl.truststore.path+:: Path to the truststore file. +{ssl-prefix}.ssl.truststore.password+:: Password to the truststore. ++{ssl-prefix}.ssl.truststore.secure_password+ (<>):: +Password to the truststore. + ===== PKCS#12 Files {security} can be configured to use PKCS#12 container files (`.p12` or `.pfx` files) @@ -110,10 +123,16 @@ Set this to `PKCS12` to indicate that the keystore is a PKCS#12 file. +{ssl-prefix}.ssl.keystore.password+:: Password to the PKCS#12 file. ++{ssl-prefix}.ssl.keystore.secure_password+ (<>):: +Password to the PKCS#12 file. + +{ssl-prefix}.ssl.keystore.key_password+:: Password for the private key stored in the PKCS#12 file. Defaults to the same value as +{ssl-prefix}.ssl.keystore.password+. ++{ssl-prefix}.ssl.keystore.secure_key_password+ (<>):: +Password for the private key stored in the PKCS#12 file. + +{ssl-prefix}.ssl.truststore.path+:: Path to the PKCS#12 file that holds the certificates to be trusted. @@ -121,4 +140,7 @@ Path to the PKCS#12 file that holds the certificates to be trusted. Set this to `PKCS12` to indicate that the truststore is a PKCS#12 file. +{ssl-prefix}.ssl.truststore.password+:: -Password to the PKCS#12 file. \ No newline at end of file +Password to the PKCS#12 file. + ++{ssl-prefix}.ssl.truststore.secure_password+ (<>):: +Password to the PKCS#12 file. From cdaa047d56477a53f9dd3933dbbc8539fd986918 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 24 Oct 2017 10:38:29 -0600 Subject: [PATCH 08/10] [TEST] Fix compilation for ARS stats exposing Original commit: elastic/x-pack-elasticsearch@eb0ad994342dee86d017197a492d44e469ebbc4d --- .../xpack/deprecation/DeprecationInfoActionResponseTests.java | 2 +- .../monitoring/collector/node/NodeStatsMonitoringDocTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index b79ff697063..504cf6510a6 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -79,7 +79,7 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa null, null, null, null, null, null)); List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, null, null, null, null, null, null, null, null, - null, null, null)); + null, null, null, null)); IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY); IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java index dd0d1bbf04f..7d8e17fb176 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java @@ -372,6 +372,6 @@ public class NodeStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCa emptySet(), Version.V_6_0_0_beta1); - return new NodeStats(discoveryNode, no, indices, os, process, jvm, threadPool, fs, null, null, null, null, null, null); + return new NodeStats(discoveryNode, no, indices, os, process, jvm, threadPool, fs, null, null, null, null, null, null, null); } } From c67ec73cb50ece36ce36fd57e9e9168da6e65628 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 25 Oct 2017 09:00:53 -0700 Subject: [PATCH 09/10] [DOCS] Add xpack.ml.max_model_memory_limit (elastic/x-pack-elasticsearch#2787) * [DOCS] Add xpack.ml.max.model_memory_limit * [DOCS] Addressed feedback on model limit setting Original commit: elastic/x-pack-elasticsearch@77a10bfe0ea3bac75831b1788f85dd92f7af1406 --- docs/en/rest-api/ml/jobresource.asciidoc | 24 +++++++++++++------- docs/en/settings/ml-settings.asciidoc | 29 +++++++++++++++--------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/en/rest-api/ml/jobresource.asciidoc b/docs/en/rest-api/ml/jobresource.asciidoc index 2fc04572dc6..1710fe3cadf 100644 --- a/docs/en/rest-api/ml/jobresource.asciidoc +++ b/docs/en/rest-api/ml/jobresource.asciidoc @@ -325,14 +325,22 @@ For more information, see This value will need to be increased for jobs that are expected to analyze high cardinality fields, but the default is set to a relatively small size to ensure that high resource usage is a conscious decision. The default value for jobs - created in versions earlier than 6.1 is `4096mb`. If you specify a number instead - of a string, the units are assumed to be MiB. Specifying a string is recommended - for clarity. If you specify a byte size unit of `b` or `kb` and the number - does not equate to a discrete number of megabytes, it is rounded down to the - closest MiB. The minimum valid value is 1 MiB. If you specify a value less - than 1 MiB, an error occurs. For more information about supported byte size - units, see - {ref}/common-options.html#byte-units[Byte size units]. + created in versions earlier than 6.1 is `4096mb`. ++ +-- +If you specify a number instead of a string, the units are assumed to be MiB. +Specifying a string is recommended for clarity. If you specify a byte size unit +of `b` or `kb` and the number does not equate to a discrete number of megabytes, +it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you +specify a value less than 1 MiB, an error occurs. For more information about +supported byte size units, see +{ref}/common-options.html#byte-units[Byte size units]. + +If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit` +setting, an error occurs when you try to create jobs that have +`model_memory_limit` values greater than that setting. For more information, +see <>. +-- [float] [[ml-apimodelplotconfig]] diff --git a/docs/en/settings/ml-settings.asciidoc b/docs/en/settings/ml-settings.asciidoc index c40de482fcf..bedc09c6332 100644 --- a/docs/en/settings/ml-settings.asciidoc +++ b/docs/en/settings/ml-settings.asciidoc @@ -11,6 +11,17 @@ You do not need to configure any settings to use {ml}. It is enabled by default. [[general-ml-settings]] ==== General Machine Learning Settings +`node.ml`:: +Set to `true` (default) to identify the node as a _machine learning node_. + ++ +If set to `false` in `elasticsearch.yml`, the node cannot run jobs. If set to +`true` but `xpack.ml.enabled` is set to `false`, the `node.ml` setting is +ignored and the node cannot run jobs. If you want to run jobs, there must be at +least one machine learning node in your cluster. + ++ +IMPORTANT: On dedicated coordinating nodes or dedicated master nodes, disable +the `node.ml` role. + `xpack.ml.enabled`:: Set to `true` (default) to enable {ml} on the node. + + @@ -26,16 +37,12 @@ IMPORTANT: If you want to use {ml} features in your cluster, you must have `xpack.ml.enabled` set to `true` on all master-eligible nodes. This is the default behavior. -`node.ml`:: -Set to `true` (default) to identify the node as a _machine learning node_. + -+ -If set to `false` in `elasticsearch.yml`, the node cannot run jobs. If set to -`true` but `xpack.ml.enabled` is set to `false`, the `node.ml` setting is -ignored and the node cannot run jobs. If you want to run jobs, there must be at -least one machine learning node in your cluster. + -+ -IMPORTANT: On dedicated coordinating nodes or dedicated master nodes, disable -the `node.ml` role. - `xpack.ml.max_open_jobs`:: The maximum number of jobs that can run on a node. Defaults to `10`. + +`xpack.ml.max_model_memory_limit`:: +The maximum `model_memory_limit` property value that can be set for any job on +this node. If you try to create a job with a `model_memory_limit` property value +that is greater than this setting value, an error occurs. Existing jobs are not +affected when you update this setting. For more information about the +`model_memory_limit` property, see <>. From 615de687110f1fcf95c99c16df63a597b6fe2755 Mon Sep 17 00:00:00 2001 From: lcawley Date: Wed, 25 Oct 2017 16:35:20 -0700 Subject: [PATCH 10/10] [DOCS] Added xpack.watcher.history.cleaner_service_enabled Original commit: elastic/x-pack-elasticsearch@ceead2a1c2896ea8a7c1de7f63c88810cbfc15eb --- docs/en/settings/notification-settings.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/en/settings/notification-settings.asciidoc b/docs/en/settings/notification-settings.asciidoc index a3727a82afd..9432cf96b63 100644 --- a/docs/en/settings/notification-settings.asciidoc +++ b/docs/en/settings/notification-settings.asciidoc @@ -16,6 +16,11 @@ Slack>>, and <>. `xpack.watcher.enabled`:: Set to `false` to disable {watcher} on the node. +`xpack.watcher.history.cleaner_service.enabled`:: +Set to `false` (default) to disable the cleaner service, which removes previous +versions of {watcher} indices (for example, .watcher-history*) when it +determines that they are old. + `xpack.http.proxy.host`:: Specifies the address of the proxy server to use to connect to HTTP services.