honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Allocate new indices on "hot" or "content" tier depending on data stream inclusion (#62338) (#62557) 

Backports the following commits to 7.x:

    Allocate new indices on "hot" or "content" tier depending on data stream inclusion (#62338)
This commit is contained in:
Lee Hinman 2020-09-17 13:29:23 -06:00 committed by GitHub
parent 1b0e6caae6
commit 9bb7ce0b22
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 130 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/api-conventions.asciidoc
View File

@ -396,7 +396,7 @@ Returns:
"index.creation_date": "1474389951325",
"index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
"index.version.created": ...,
"index.routing.allocation.include._tier" : "data_hot",
"index.routing.allocation.include._tier" : "data_content",
"index.provided_name" : "my-index-000001"
}
}
@ -433,7 +433,7 @@ Returns:
"routing": {
"allocation": {
"include": {
"_tier": "data_hot"
"_tier": "data_content"
}
}
},

4
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View File

@ -675,10 +675,12 @@ public class MetadataCreateIndexService {
.put(request.settings())
.build();
final boolean isDataStreamIndex = request.dataStreamName() != null;
// Loop through all the explicit index setting providers, adding them to the
// additionalIndexSettings map
for (IndexSettingProvider provider : indexSettingProviders) {
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), templateAndRequestSettings));
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(),
isDataStreamIndex, templateAndRequestSettings));
}
// For all the explicit settings, we go through the template and request level settings

2
server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java
View File

@ -30,7 +30,7 @@ public interface IndexSettingProvider {
* Returns explicitly set default index {@link Settings} for the given index. This should not
* return null.
*/
default Settings getAdditionalIndexSettings(String indexName, Settings templateAndRequestSettings) {
default Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
return Settings.EMPTY;
}
}

29
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java → x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java
View File

@ -24,7 +24,7 @@ import java.util.Collections;
import static org.hamcrest.Matchers.equalTo;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0)
public class DataTierIT extends ESIntegTestCase {
private static final String index = "myindex";
@ -33,7 +33,7 @@ public class DataTierIT extends ESIntegTestCase {
return Collections.singleton(LocalStateCompositeXPackPlugin.class);
}
public void testDefaultAllocateToHot() {
public void testDefaultIndexAllocateToContent() {
startWarmOnlyNode();
startColdOnlyNode();
ensureGreen();
@ -41,14 +41,19 @@ public class DataTierIT extends ESIntegTestCase {
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
// index should be red
assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
equalTo(ClusterHealthStatus.RED));
logger.info("--> starting hot node");
startHotOnlyNode();
if (randomBoolean()) {
logger.info("--> starting content node");
startContentOnlyNode();
} else {
logger.info("--> starting hot node");
startDataNode();
}
logger.info("--> waiting for {} to be yellow", index);
ensureYellow(index);
@ -189,6 +194,20 @@ public class DataTierIT extends ESIntegTestCase {
ensureYellow(index);
}
public void startDataNode() {
Settings nodeSettings = Settings.builder()
.putList("node.roles", Arrays.asList("master", "data", "ingest"))
.build();
internalCluster().startNode(nodeSettings);
}
public void startContentOnlyNode() {
Settings nodeSettings = Settings.builder()
.putList("node.roles", Arrays.asList("master", "data_content", "ingest"))
.build();
internalCluster().startNode(nodeSettings);
}
public void startHotOnlyNode() {
Settings nodeSettings = Settings.builder()
.putList("node.roles", Arrays.asList("master", "data_hot", "ingest"))

14
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java
View File

@ -154,7 +154,7 @@ public class DataTier {
private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class);
@Override
public Settings getAdditionalIndexSettings(String indexName, Settings indexSettings) {
public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings indexSettings) {
Set<String> settings = indexSettings.keySet();
if (settings.contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) {
// It's okay to put it, it will be removed or overridden by the template/request settings
@ -163,11 +163,17 @@ public class DataTier {
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) ||
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) {
// A different index level require, include, or exclude has been specified, so don't put the setting
logger.debug("index [{}] specifies custom index level routing filtering, skipping hot tier allocation", indexName);
logger.debug("index [{}] specifies custom index level routing filtering, skipping tier allocation", indexName);
return Settings.EMPTY;
} else {
// Otherwise, put the setting in place by default
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
// Otherwise, put the setting in place by default, the "hot"
// tier if the index is part of a data stream, the "content"
// tier if it is not.
if (isDataStreamIndex) {
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
} else {
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_CONTENT).build();
}
}
}
}

90
x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.datastreams;
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
import org.elasticsearch.xpack.core.DataTier;
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import static org.hamcrest.Matchers.equalTo;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0)
public class DataTierDataStreamIT extends ESIntegTestCase {
private static final String index = "myindex";
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(LocalStateCompositeXPackPlugin.class, DataStreamsPlugin.class);
}
public void testDefaultDataStreamAllocateToHot() {
startHotOnlyNode();
ensureGreen();
ComposableIndexTemplate template = new ComposableIndexTemplate(
Collections.singletonList(index),
null,
null,
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate()
);
client().execute(
PutComposableIndexTemplateAction.INSTANCE,
new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)
).actionGet();
client().prepareIndex(index, MapperService.SINGLE_MAPPING_NAME)
.setCreate(true)
.setId("1")
.setSource("@timestamp", "2020-09-09")
.setWaitForActiveShards(0)
.get();
Settings idxSettings = client().admin()
.indices()
.prepareGetIndex()
.addIndices(index)
.get()
.getSettings()
.get(DataStream.getDefaultBackingIndexName(index, 1));
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
logger.info("--> waiting for {} to be yellow", index);
ensureYellow(index);
// Roll over index and ensure the second index also went to the "hot" tier
client().admin().indices().prepareRolloverIndex(index).get();
idxSettings = client().admin()
.indices()
.prepareGetIndex()
.addIndices(index)
.get()
.getSettings()
.get(DataStream.getDefaultBackingIndexName(index, 2));
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { index }));
}
public void startHotOnlyNode() {
Settings nodeSettings = Settings.builder().putList("node.roles", Arrays.asList("master", "data_hot", "ingest")).build();
internalCluster().startNode(nodeSettings);
}
}