honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

System index reads in separate threadpool (#60927) 

This commit introduces a new thread pool, `system_read`, which is
intended for use by system indices for all read operations (get and
search). The `system_read` pool is a fixed thread pool with a maximum
number of threads equal to lesser of half of the available processors
or 5. Given the combination of both get and read operations in this
thread pool, the queue size has been set to 2000. The motivation for
this change is to allow system read operations to be serviced in spite
of the number of user searches.

In order to avoid a significant performance hit due to pattern matching
on all search requests, a new metadata flag is added to mark indices
as system or non-system. Previously created system indices will have
flag added to their metadata upon upgrade to a version with this
capability.

Additionally, this change also introduces a new class, `SystemIndices`,
which encapsulates logic around system indices. Currently, the class
provides a method to check if an index is a system index and a method
to find a matching index descriptor given the name of an index.

Relates #50251
Relates #37867
Backport of #57936
This commit is contained in:
Jay Modi 2020-08-11 12:16:34 -06:00 committed by GitHub
parent a93be8d577
commit 2fa6448a15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 759 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/reference/modules/threadpool.asciidoc
View File

@ -1,9 +1,9 @@
[[modules-threadpool]]
=== Thread pools
A node uses several thread pools to manage memory consumption.
Queues associated with many of the thread pools enable pending requests
to be held instead of discarded.
A node uses several thread pools to manage memory consumption.
Queues associated with many of the thread pools enable pending requests
to be held instead of discarded.
There are several thread pools, but the important ones include:
@ -83,6 +83,11 @@ There are several thread pools, but the important ones include:
Thread pool type is `scaling` with a keep-alive of `5m` and a default
maximum size of `5`.
`system_read`::
For read operations on system indices.
Thread pool type is `fixed` and a default maximum size of
`min(5, (`<<node.processors, `# of allocated processors`>>`) / 2)`.
Changing a specific thread pool can be done by setting its type-specific
parameters; for example, changing the number of threads in the `write` thread
pool:

69
qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
View File

@ -1457,6 +1457,75 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
}
public void testCreateSystemIndexInOldVersion() throws Exception {
assumeTrue("only run on old cluster", isRunningAgainstOldCluster());
// create index
Request createTestIndex = new Request("PUT", "/test_index_old");
createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0, \"index.number_of_shards\": 1}}");
client().performRequest(createTestIndex);
Request bulk = new Request("POST", "/_bulk");
bulk.addParameter("refresh", "true");
bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
"{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
if (isRunningAgainstAncientCluster() == false) {
bulk.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE));
}
client().performRequest(bulk);
// start a async reindex job
Request reindex = new Request("POST", "/_reindex");
reindex.setJsonEntity(
"{\n" +
" \"source\":{\n" +
" \"index\":\"test_index_old\"\n" +
" },\n" +
" \"dest\":{\n" +
" \"index\":\"test_index_reindex\"\n" +
" }\n" +
"}");
reindex.addParameter("wait_for_completion", "false");
Map<String, Object> response = entityAsMap(client().performRequest(reindex));
String taskId = (String) response.get("task");
// wait for task
Request getTask = new Request("GET", "/_tasks/" + taskId);
getTask.addParameter("wait_for_completion", "true");
client().performRequest(getTask);
// make sure .tasks index exists
assertBusy(() -> {
Request getTasksIndex = new Request("GET", "/.tasks");
if (isRunningAgainstAncientCluster()) {
getTasksIndex.addParameter("include_type_name", "false");
}
assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
});
}
@SuppressWarnings("unchecked" +
"")
public void testSystemIndexGetsUpdatedMetadata() throws Exception {
assumeFalse("only run in upgraded cluster", isRunningAgainstOldCluster());
assertBusy(() -> {
Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
Map<String, Object> response = entityAsMap(client().performRequest(clusterStateRequest));
Map<String, Object> metadata = (Map<String, Object>) response.get("metadata");
assertNotNull(metadata);
Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
assertNotNull(indices);
Map<String, Object> tasksIndex = (Map<String, Object>) indices.get(".tasks");
assertNotNull(tasksIndex);
assertThat(tasksIndex.get("system"), is(true));
Map<String, Object> testIndex = (Map<String, Object>) indices.get("test_index_old");
assertNotNull(testIndex);
assertThat(testIndex.get("system"), is(false));
});
}
public static void assertNumHits(String index, int numHits, int totalShards) throws IOException {
Map<String, Object> resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
assertNoFailures(resp);

11
qa/rolling-upgrade/build.gradle
View File

@ -41,10 +41,11 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
* </ul>
*/
String baseName = "v${bwcVersion}"
String bwcVersionStr = "${bwcVersion}"
testClusters {
"${baseName}" {
versions = [bwcVersion.toString(), project.version]
versions = [bwcVersionStr, project.version]
numberOfNodes = 3
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
@ -60,7 +61,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
doFirst {
project.delete("${buildDir}/cluster/shared/repo/${baseName}")
}
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
systemProperty 'tests.upgrade_from_version', bwcVersionStr
systemProperty 'tests.rest.suite', 'old_cluster'
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@ -73,7 +74,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
testClusters."${baseName}".nextNodeToNextVersion()
}
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
systemProperty 'tests.upgrade_from_version', bwcVersionStr
systemProperty 'tests.first_round', 'true'
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@ -86,7 +87,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
testClusters."${baseName}".nextNodeToNextVersion()
}
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
systemProperty 'tests.upgrade_from_version', bwcVersionStr
systemProperty 'tests.first_round', 'false'
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@ -99,7 +100,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
}
useCluster testClusters."${baseName}"
systemProperty 'tests.rest.suite', 'upgraded_cluster'
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
systemProperty 'tests.upgrade_from_version', bwcVersionStr
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")

2
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MappingIT.java
View File

@ -18,11 +18,13 @@
*/
package org.elasticsearch.upgrades;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/60986")
public class MappingIT extends AbstractRollingTestCase {
/**
* Create a mapping that explicitly disables the _all field (possible in 6x, see #37429)

104
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.upgrades;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import java.util.Map;
import static org.hamcrest.Matchers.is;
public class SystemIndicesUpgradeIT extends AbstractRollingTestCase {
public void testOldDoesntHaveSystemIndexMetadata() throws Exception {
assumeTrue("only run in old cluster", CLUSTER_TYPE == ClusterType.OLD);
// create index
Request createTestIndex = new Request("PUT", "/test_index_old");
createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0, \"index.number_of_shards\": 1}}");
client().performRequest(createTestIndex);
Request bulk = new Request("POST", "/_bulk");
bulk.addParameter("refresh", "true");
if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
"{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
} else {
bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\"}\n" +
"{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
}
client().performRequest(bulk);
// start a async reindex job
Request reindex = new Request("POST", "/_reindex");
reindex.setJsonEntity(
"{\n" +
" \"source\":{\n" +
" \"index\":\"test_index_old\"\n" +
" },\n" +
" \"dest\":{\n" +
" \"index\":\"test_index_reindex\"\n" +
" }\n" +
"}");
reindex.addParameter("wait_for_completion", "false");
Map<String, Object> response = entityAsMap(client().performRequest(reindex));
String taskId = (String) response.get("task");
// wait for task
Request getTask = new Request("GET", "/_tasks/" + taskId);
getTask.addParameter("wait_for_completion", "true");
client().performRequest(getTask);
// make sure .tasks index exists
assertBusy(() -> {
Request getTasksIndex = new Request("GET", "/.tasks");
if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
getTasksIndex.addParameter("include_type_name", "false");
}
assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
});
}
public void testMixedCluster() {
assumeTrue("nothing to do in mixed cluster", CLUSTER_TYPE == ClusterType.MIXED);
}
@SuppressWarnings("unchecked")
public void testUpgradedCluster() throws Exception {
assumeTrue("only run on upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED);
assertBusy(() -> {
Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
Map<String, Object> response = entityAsMap(client().performRequest(clusterStateRequest));
Map<String, Object> metadata = (Map<String, Object>) response.get("metadata");
assertNotNull(metadata);
Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
assertNotNull(indices);
Map<String, Object> tasksIndex = (Map<String, Object>) indices.get(".tasks");
assertNotNull(tasksIndex);
assertThat(tasksIndex.get("system"), is(true));
Map<String, Object> testIndex = (Map<String, Object>) indices.get("test_index_old");
assertNotNull(testIndex);
assertThat(testIndex.get("system"), is(false));
});
}
}

4
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
View File

@ -95,7 +95,9 @@
index: .tasks
body:
query:
match_all: {}
match:
task.description:
query: 'reindexed_index_copy'
- match: { hits.total: 1 }
- match: { hits.hits.0._id: '/.+:\d+/' }
- set: {hits.hits.0._id: task_id}

11
server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
View File

@ -115,8 +115,13 @@ public class TransportGetAction extends TransportSingleShardAction<GetRequest, G
@Override
protected String getExecutor(GetRequest request, ShardId shardId) {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
return indexService.getIndexSettings().isSearchThrottled() ? ThreadPool.Names.SEARCH_THROTTLED : super.getExecutor(request,
shardId);
final ClusterState clusterState = clusterService.state();
if (clusterState.metadata().index(shardId.getIndex()).isSystem()) {
return ThreadPool.Names.SYSTEM_READ;
} else if (indicesService.indexServiceSafe(shardId.getIndex()).getIndexSettings().isSearchThrottled()) {
return ThreadPool.Names.SEARCH_THROTTLED;
} else {
return super.getExecutor(request, shardId);
}
}
}

11
server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
View File

@ -126,8 +126,13 @@ public class TransportShardMultiGetAction extends TransportSingleShardAction<Mul
@Override
protected String getExecutor(MultiGetShardRequest request, ShardId shardId) {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
return indexService.getIndexSettings().isSearchThrottled() ? ThreadPool.Names.SEARCH_THROTTLED : super.getExecutor(request,
shardId);
final ClusterState clusterState = clusterService.state();
if (clusterState.metadata().index(shardId.getIndex()).isSystem()) {
return ThreadPool.Names.SYSTEM_READ;
} else if (indicesService.indexServiceSafe(shardId.getIndex()).getIndexSettings().isSearchThrottled()) {
return ThreadPool.Names.SEARCH_THROTTLED;
} else {
return super.getExecutor(request, shardId);
}
}
}

12
server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
View File

@ -519,10 +519,17 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
nodes::get, remoteConnections, searchTransportService::getConnection);
boolean preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, indices, shardIterators.size());
searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
final Executor asyncSearchExecutor = asyncSearchExecutor(indices, clusterState);
searchAsyncAction(task, searchRequest, asyncSearchExecutor, shardIterators, timeProvider, connectionLookup, clusterState,
Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start();
}
Executor asyncSearchExecutor(final Index[] indices, final ClusterState clusterState) {
final boolean onlySystemIndices =
Arrays.stream(indices).allMatch(index -> clusterState.metadata().index(index.getName()).isSystem());
return onlySystemIndices ? threadPool.executor(ThreadPool.Names.SYSTEM_READ) : threadPool.executor(ThreadPool.Names.SEARCH);
}
static BiFunction<String, String, Transport.Connection> buildConnectionLookup(String requestClusterAlias,
Function<String, DiscoveryNode> localNodes,
BiFunction<String, String, DiscoveryNode> remoteNodes,
@ -584,6 +591,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
}
private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction(SearchTask task, SearchRequest searchRequest,
Executor executor,
GroupShardsIterator<SearchShardIterator> shardIterators,
SearchTimeProvider timeProvider,
BiFunction<String, String, Transport.Connection> connectionLookup,
@ -594,7 +602,6 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
ActionListener<SearchResponse> listener,
boolean preFilter,
SearchResponse.Clusters clusters) {
Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
if (preFilter) {
return new CanMatchPreFilterSearchPhase(logger, searchTransportService, connectionLookup,
aliasFilter, concreteIndexBoosts, indexRoutings, executor, searchRequest, listener, shardIterators,
@ -602,6 +609,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
AbstractSearchAsyncAction<? extends SearchPhaseResult> action = searchAsyncAction(
task,
searchRequest,
executor,
iter,
timeProvider,
connectionLookup,

52
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View File

@ -340,9 +340,13 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
static final String KEY_MAPPINGS = "mappings";
static final String KEY_ALIASES = "aliases";
static final String KEY_ROLLOVER_INFOS = "rollover_info";
static final String KEY_SYSTEM = "system";
public static final String KEY_PRIMARY_TERMS = "primary_terms";
public static final String INDEX_STATE_FILE_PREFIX = "state-";
static final Version SYSTEM_INDEX_FLAG_ADDED = Version.V_7_10_0;
private final int routingNumShards;
private final int routingFactor;
private final int routingPartitionSize;
@ -385,6 +389,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
private final ActiveShardCount waitForActiveShards;
private final ImmutableOpenMap<String, RolloverInfo> rolloverInfos;
private final boolean isSystem;
private IndexMetadata(
final Index index,
@ -410,7 +415,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
final int routingNumShards,
final int routingPartitionSize,
final ActiveShardCount waitForActiveShards,
final ImmutableOpenMap<String, RolloverInfo> rolloverInfos) {
final ImmutableOpenMap<String, RolloverInfo> rolloverInfos,
final boolean isSystem) {
this.index = index;
this.version = version;
@ -442,6 +448,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
this.routingPartitionSize = routingPartitionSize;
this.waitForActiveShards = waitForActiveShards;
this.rolloverInfos = rolloverInfos;
this.isSystem = isSystem;
assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
}
@ -703,6 +710,9 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
if (rolloverInfos.equals(that.rolloverInfos) == false) {
return false;
}
if (isSystem != that.isSystem) {
return false;
}
return true;
}
@ -720,6 +730,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
result = 31 * result + Arrays.hashCode(primaryTerms);
result = 31 * result + inSyncAllocationIds.hashCode();
result = 31 * result + rolloverInfos.hashCode();
result = 31 * result + Boolean.hashCode(isSystem);
return result;
}
@ -759,6 +770,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
private final Diff<ImmutableOpenMap<String, DiffableStringMap>> customData;
private final Diff<ImmutableOpenIntMap<Set<String>>> inSyncAllocationIds;
private final Diff<ImmutableOpenMap<String, RolloverInfo>> rolloverInfos;
private final boolean isSystem;
IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
index = after.index.getName();
@ -776,6 +788,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
inSyncAllocationIds = DiffableUtils.diff(before.inSyncAllocationIds, after.inSyncAllocationIds,
DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer());
isSystem = after.isSystem;
}
IndexMetadataDiff(StreamInput in) throws IOException {
@ -815,6 +828,11 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
ImmutableOpenMap<String, RolloverInfo> emptyMap = ImmutableOpenMap.of();
rolloverInfos = DiffableUtils.diff(emptyMap, emptyMap, DiffableUtils.getStringKeySerializer());
}
if (in.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
isSystem = in.readBoolean();
} else {
isSystem = false;
}
}
@Override
@ -841,6 +859,9 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
rolloverInfos.writeTo(out);
}
if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
out.writeBoolean(isSystem);
}
}
@Override
@ -859,6 +880,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
builder.customMetadata.putAll(customData.apply(part.customData));
builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds));
builder.rolloverInfos.putAll(rolloverInfos.apply(part.rolloverInfos));
builder.system(part.isSystem);
return builder.build();
}
}
@ -920,6 +942,9 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
builder.putRolloverInfo(new RolloverInfo(in));
}
}
if (in.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
builder.system(in.readBoolean());
}
return builder.build();
}
@ -968,6 +993,13 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
cursor.value.writeTo(out);
}
}
if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
out.writeBoolean(isSystem);
}
}
public boolean isSystem() {
return isSystem;
}
public static Builder builder(String index) {
@ -994,6 +1026,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
private final ImmutableOpenIntMap.Builder<Set<String>> inSyncAllocationIds;
private final ImmutableOpenMap.Builder<String, RolloverInfo> rolloverInfos;
private Integer routingNumShards;
private boolean isSystem;
public Builder(String index) {
this.index = index;
@ -1002,6 +1035,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
this.customMetadata = ImmutableOpenMap.builder();
this.inSyncAllocationIds = ImmutableOpenIntMap.builder();
this.rolloverInfos = ImmutableOpenMap.builder();
this.isSystem = false;
}
public Builder(IndexMetadata indexMetadata) {
@ -1019,6 +1053,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
this.routingNumShards = indexMetadata.routingNumShards;
this.inSyncAllocationIds = ImmutableOpenIntMap.builder(indexMetadata.inSyncAllocationIds);
this.rolloverInfos = ImmutableOpenMap.builder(indexMetadata.rolloverInfos);
this.isSystem = indexMetadata.isSystem;
}
public Builder index(String index) {
@ -1218,6 +1253,15 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
Arrays.fill(primaryTerms, SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
}
public Builder system(boolean system) {
this.isSystem = system;
return this;
}
public boolean isSystem() {
return isSystem;
}
public IndexMetadata build() {
ImmutableOpenMap.Builder<String, AliasMetadata> tmpAliases = aliases;
Settings tmpSettings = settings;
@ -1330,7 +1374,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
getRoutingNumShards(),
routingPartitionSize,
waitForActiveShards,
rolloverInfos.build());
rolloverInfos.build(),
isSystem);
}
public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
@ -1429,6 +1474,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
cursor.value.toXContent(builder, params);
}
builder.endObject();
builder.field(KEY_SYSTEM, indexMetadata.isSystem);
builder.endObject();
}
@ -1556,6 +1602,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
builder.aliasesVersion(parser.longValue());
} else if (KEY_ROUTING_NUM_SHARDS.equals(currentFieldName)) {
builder.setRoutingNumShards(parser.intValue());
} else if (KEY_SYSTEM.equals(currentFieldName)) {
builder.system(parser.booleanValue());
} else {
throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
}

48
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View File

@ -74,6 +74,7 @@ import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
@ -82,7 +83,6 @@ import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -127,7 +127,7 @@ public class MetadataCreateIndexService {
private final IndexScopedSettings indexScopedSettings;
private final ActiveShardsObserver activeShardsObserver;
private final NamedXContentRegistry xContentRegistry;
private final Collection<SystemIndexDescriptor> systemIndexDescriptors;
private final SystemIndices systemIndices;
private final ShardLimitValidator shardLimitValidator;
private final boolean forbidPrivateIndexSettings;
@ -142,7 +142,7 @@ public class MetadataCreateIndexService {
final IndexScopedSettings indexScopedSettings,
final ThreadPool threadPool,
final NamedXContentRegistry xContentRegistry,
final Collection<SystemIndexDescriptor> systemIndexDescriptors,
final SystemIndices systemIndices,
final boolean forbidPrivateIndexSettings) {
this.settings = settings;
this.clusterService = clusterService;
@ -153,7 +153,7 @@ public class MetadataCreateIndexService {
this.indexScopedSettings = indexScopedSettings;
this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool);
this.xContentRegistry = xContentRegistry;
this.systemIndexDescriptors = systemIndexDescriptors;
this.systemIndices = systemIndices;
this.forbidPrivateIndexSettings = forbidPrivateIndexSettings;
this.shardLimitValidator = shardLimitValidator;
}
@ -183,32 +183,26 @@ public class MetadataCreateIndexService {
/**
* Validates (if this index has a dot-prefixed name) whether it follows the rules for dot-prefixed indices.
* @param index The name of the index in question
* @param state The current cluster state
* @param isHidden Whether or not this is a hidden index
*/
public void validateDotIndex(String index, ClusterState state, @Nullable Boolean isHidden) {
public boolean validateDotIndex(String index, @Nullable Boolean isHidden) {
boolean isSystem = false;
if (index.charAt(0) == '.') {
List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
.filter(descriptor -> descriptor.matchesIndexPattern(index))
.collect(toList());
if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
SystemIndexDescriptor matchingDescriptor = systemIndices.findMatchingDescriptor(index);
if (matchingDescriptor != null) {
logger.trace("index [{}] is a system index because it matches index pattern [{}] with description [{}]",
index, matchingDescriptor.getIndexPattern(), matchingDescriptor.getDescription());
isSystem = true;
} else if (isHidden) {
logger.trace("index [{}] is a hidden index", index);
} else {
DEPRECATION_LOGGER.deprecatedAndMaybeLog("index_name_starts_with_dot",
"index name [{}] starts with a dot '.', in the next major version, index names " +
"starting with a dot are reserved for hidden indices and system indices", index);
} else if (matchingDescriptors.size() > 1) {
// This should be prevented by erroring on overlapping patterns at startup time, but is here just in case.
StringBuilder errorMessage = new StringBuilder()
.append("index name [")
.append(index)
.append("] is claimed as a system index by multiple system index patterns: [")
.append(matchingDescriptors.stream()
.map(descriptor -> "pattern: [" + descriptor.getIndexPattern() +
"], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; ")));
// Throw AssertionError if assertions are enabled, or a regular exception otherwise:
assert false : errorMessage.toString();
throw new IllegalStateException(errorMessage.toString());
"starting with a dot are reserved for hidden indices and system indices", index);
}
}
return isSystem;
}
/**
@ -408,7 +402,7 @@ public class MetadataCreateIndexService {
try {
indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper,
() -> indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING), temporaryIndexMeta.getSettings(),
temporaryIndexMeta.getRoutingNumShards(), sourceMetadata);
temporaryIndexMeta.getRoutingNumShards(), sourceMetadata, temporaryIndexMeta.isSystem());
} catch (Exception e) {
logger.info("failed to build index metadata [{}]", request.index());
throw e;
@ -433,7 +427,7 @@ public class MetadataCreateIndexService {
final int routingNumShards) {
final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings);
validateDotIndex(request.index(), currentState, isHiddenAfterTemplates);
final boolean isSystem = validateDotIndex(request.index(), isHiddenAfterTemplates);
// remove the setting it's temporary and is only relevant once we create the index
final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings);
@ -443,6 +437,7 @@ public class MetadataCreateIndexService {
final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index());
tmpImdBuilder.setRoutingNumShards(routingNumShards);
tmpImdBuilder.settings(indexSettings);
tmpImdBuilder.system(isSystem);
// Set up everything, now locally create the index to see that things are ok, and apply
IndexMetadata tempMetadata = tmpImdBuilder.build();
@ -829,8 +824,9 @@ public class MetadataCreateIndexService {
static IndexMetadata buildIndexMetadata(String indexName, List<AliasMetadata> aliases,
Supplier<DocumentMapper> documentMapperSupplier,
Supplier<DocumentMapper> defaultDocumentMapperSupplier, Settings indexSettings,
int routingNumShards, @Nullable IndexMetadata sourceMetadata) {
int routingNumShards, @Nullable IndexMetadata sourceMetadata, boolean isSystem) {
IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
indexMetadataBuilder.system(isSystem);
// now, update the mappings with the actual source
Map<String, MappingMetadata> mappingsMetadata = new HashMap<>();
for (DocumentMapper mapper : Arrays.asList(documentMapperSupplier.get(), defaultDocumentMapperSupplier.get())) {

24
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.script.ScriptService;
@ -59,13 +60,15 @@ public class MetadataIndexUpgradeService {
private final NamedXContentRegistry xContentRegistry;
private final MapperRegistry mapperRegistry;
private final IndexScopedSettings indexScopedSettings;
private final SystemIndices systemIndices;
public MetadataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
IndexScopedSettings indexScopedSettings) {
IndexScopedSettings indexScopedSettings, SystemIndices systemIndices) {
this.settings = settings;
this.xContentRegistry = xContentRegistry;
this.mapperRegistry = mapperRegistry;
this.indexScopedSettings = indexScopedSettings;
this.systemIndices = systemIndices;
}
/**
@ -76,24 +79,25 @@ public class MetadataIndexUpgradeService {
* cannot be updated the method throws an exception.
*/
public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) {
// Throws an exception if there are too-old segments:
if (isUpgraded(indexMetadata)) {
/*
* We still need to check for broken index settings since it might be that a user removed a plugin that registers a setting
* needed by this index.
* needed by this index. Additionally, the system flag could have been lost during a rolling upgrade where the previous version
* did not know about the flag.
*/
return archiveBrokenIndexSettings(indexMetadata);
return archiveBrokenIndexSettings(maybeMarkAsSystemIndex(indexMetadata));
}
// Throws an exception if there are too-old segments:
checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
final IndexMetadata metadataWithSystemMarked = maybeMarkAsSystemIndex(indexMetadata);
// we have to run this first otherwise in we try to create IndexSettings
// with broken settings and fail in checkMappingsCompatibility
final IndexMetadata newMetadata = archiveBrokenIndexSettings(indexMetadata);
final IndexMetadata newMetadata = archiveBrokenIndexSettings(metadataWithSystemMarked);
// only run the check with the upgraded settings!!
checkMappingsCompatibility(newMetadata);
return markAsUpgraded(newMetadata);
}
/**
* Checks if the index was already opened by this version of Elasticsearch and doesn't require any additional checks.
*/
@ -216,4 +220,12 @@ public class MetadataIndexUpgradeService {
return indexMetadata;
}
}
IndexMetadata maybeMarkAsSystemIndex(IndexMetadata indexMetadata) {
final boolean isSystem = systemIndices.isSystemIndex(indexMetadata.getIndex());
if (isSystem != indexMetadata.isSystem()) {
return IndexMetadata.builder(indexMetadata).system(isSystem).build();
}
return indexMetadata;
}
}

114
server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.indices.SystemIndices;
import java.util.ArrayList;
import java.util.List;
/**
* A service responsible for updating the metadata used by system indices.
*/
public class SystemIndexMetadataUpgradeService implements ClusterStateListener {
private static final Logger logger = LogManager.getLogger(SystemIndexMetadataUpgradeService.class);
private final SystemIndices systemIndices;
private final ClusterService clusterService;
private boolean master = false;
private volatile ImmutableOpenMap<String, IndexMetadata> lastIndexMetadataMap = ImmutableOpenMap.of();
private volatile boolean updateTaskPending = false;
public SystemIndexMetadataUpgradeService(SystemIndices systemIndices, ClusterService clusterService) {
this.systemIndices = systemIndices;
this.clusterService = clusterService;
}
@Override
public void clusterChanged(ClusterChangedEvent event) {
if (event.localNodeMaster() != master) {
this.master = event.localNodeMaster();
}
if (master && updateTaskPending == false) {
final ImmutableOpenMap<String, IndexMetadata> indexMetadataMap = event.state().metadata().indices();
if (lastIndexMetadataMap != indexMetadataMap) {
for (ObjectObjectCursor<String, IndexMetadata> cursor : indexMetadataMap) {
if (cursor.value != lastIndexMetadataMap.get(cursor.key)) {
if (systemIndices.isSystemIndex(cursor.value.getIndex()) != cursor.value.isSystem()) {
updateTaskPending = true;
clusterService.submitStateUpdateTask("system_index_metadata_upgrade_service {system metadata change}",
new SystemIndexMetadataUpdateTask());
break;
}
}
}
}
}
}
public class SystemIndexMetadataUpdateTask extends ClusterStateUpdateTask {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
final ImmutableOpenMap<String, IndexMetadata> indexMetadataMap = currentState.metadata().indices();
final List<IndexMetadata> updatedMetadata = new ArrayList<>();
for (ObjectObjectCursor<String, IndexMetadata> cursor : indexMetadataMap) {
if (cursor.value != lastIndexMetadataMap.get(cursor.key)) {
if (systemIndices.isSystemIndex(cursor.value.getIndex()) != cursor.value.isSystem()) {
updatedMetadata.add(IndexMetadata.builder(cursor.value).system(!cursor.value.isSystem()).build());
}
}
}
if (updatedMetadata.isEmpty() == false) {
final Metadata.Builder builder = Metadata.builder(currentState.metadata());
updatedMetadata.forEach(idxMeta -> builder.put(idxMeta, true));
return ClusterState.builder(currentState).metadata(builder).build();
}
return currentState;
}
@Override
public void onFailure(String source, Exception e) {
updateTaskPending = false;
logger.error("failed to update system index metadata", e);
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
lastIndexMetadataMap = newState.metadata().indices();
updateTaskPending = false;
}
}
}

4
server/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -149,8 +149,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
private final Client client;
private final CircuitBreakerService circuitBreakerService;
private final IndexNameExpressionResolver expressionResolver;
private Supplier<Sort> indexSortSupplier;
private ValuesSourceRegistry valuesSourceRegistry;
private final Supplier<Sort> indexSortSupplier;
private final ValuesSourceRegistry valuesSourceRegistry;
public IndexService(
IndexSettings indexSettings,

4
server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -421,6 +421,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
return this.shardFieldData;
}
public boolean isSystem() {
return indexSettings.getIndexMetadata().isSystem();
}
/**
* USE THIS METHOD WITH CARE!
* Returns the primary term the index shard is supposed to be on. In case of primary promotion or when a replica learns about

7
server/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -232,8 +232,7 @@ public class IndicesService extends AbstractLifecycleComponent
private final EsThreadPoolExecutor danglingIndicesThreadPoolExecutor;
private final Set<Index> danglingIndicesToWrite = Sets.newConcurrentHashSet();
private final boolean nodeWriteDanglingIndicesInfo;
private ValuesSourceRegistry valuesSourceRegistry;
private final ValuesSourceRegistry valuesSourceRegistry;
@Override
protected void doStart() {
@ -645,7 +644,7 @@ public class IndicesService extends AbstractLifecycleComponent
indexCreationContext);
final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings),
directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories);
directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories);
for (IndexingOperationListener operationListener : indexingOperationListeners) {
indexModule.addIndexOperationListener(operationListener);
}
@ -716,7 +715,7 @@ public class IndicesService extends AbstractLifecycleComponent
public synchronized MapperService createIndexMapperService(IndexMetadata indexMetadata) throws IOException {
final IndexSettings idxSettings = new IndexSettings(indexMetadata, this.settings, indexScopedSettings);
final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings),
directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories);
directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories);
pluginsService.onIndexModule(indexModule);
return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
}

51
server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java
View File

@ -19,18 +19,10 @@
package org.elasticsearch.indices;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.regex.Regex;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* Describes a system index. Provides the information required to create and maintain the system index.
@ -92,49 +84,6 @@ public class SystemIndexDescriptor {
return "SystemIndexDescriptor[pattern=[" + indexPattern + "], description=[" + description + "]]";
}
/**
* Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed
* descriptors overlap with any of the other patterns. If any do, throws an exception.
*
* @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide.
* @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
*/
public static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) {
List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet().stream()
.flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
.sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
.collect(Collectors.toList());
// This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
// automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states
// per pattern should be low as well. If these assumptions change, this might need to be reworked.
sourceDescriptorPair.forEach(descriptorToCheck -> {
List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
.filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
.filter(d -> overlaps(descriptorToCheck.v2(), d.v2()))
.collect(Collectors.toList());
if (descriptorsMatchingThisPattern.isEmpty() == false) {
StringBuilder errorMessage = new StringBuilder();
errorMessage.append("a system index descriptor [")
.append(descriptorToCheck.v2())
.append("] from plugin [")
.append(descriptorToCheck.v1())
.append("] overlaps with other system index descriptors: [")
.append(descriptorsMatchingThisPattern.stream()
.map(descriptor -> descriptor.v2() + " from plugin [" + descriptor.v1() + "]")
.collect(Collectors.joining(", ")));
throw new IllegalStateException(errorMessage.toString());
}
});
}
private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) {
Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern());
Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern());
return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false;
}
// TODO: Index settings and mapping
// TODO: getThreadpool()
// TODO: Upgrade handling (reindex script?)

143
server/src/main/java/org/elasticsearch/indices/SystemIndices.java Normal file
View File

@ -0,0 +1,143 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.MinimizationOperations;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.Index;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import static java.util.Collections.unmodifiableList;
/**
* This class holds the {@link SystemIndexDescriptor} objects that represent system indices the
* node knows about. Methods for determining if an index should be a system index are also provided
* to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s.
*/
public class SystemIndices {
private final CharacterRunAutomaton runAutomaton;
private final Collection<SystemIndexDescriptor> systemIndexDescriptors;
public SystemIndices(Map<String, Collection<SystemIndexDescriptor>> systemIndexDescriptorMap) {
checkForOverlappingPatterns(systemIndexDescriptorMap);
this.systemIndexDescriptors = unmodifiableList(systemIndexDescriptorMap.values()
.stream()
.flatMap(Collection::stream)
.collect(Collectors.toList()));
this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors);
}
/**
* Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s
* @param index the {@link Index} object to check against loaded {@link SystemIndexDescriptor}s
* @return true if the {@link Index}'s name matches a pattern from a {@link SystemIndexDescriptor}
*/
public boolean isSystemIndex(Index index) {
return runAutomaton.run(index.getName());
}
/**
* Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
* @param name the name of the index
* @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
* @throws IllegalStateException if multiple descriptors match the name
*/
public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
final List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
.filter(descriptor -> descriptor.matchesIndexPattern(name))
.collect(Collectors.toList());
if (matchingDescriptors.isEmpty()) {
return null;
} else if (matchingDescriptors.size() == 1) {
return matchingDescriptors.get(0);
} else {
// This should be prevented by failing on overlapping patterns at startup time, but is here just in case.
StringBuilder errorMessage = new StringBuilder()
.append("index name [")
.append(name)
.append("] is claimed as a system index by multiple system index patterns: [")
.append(matchingDescriptors.stream()
.map(descriptor -> "pattern: [" + descriptor.getIndexPattern() +
"], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; ")));
// Throw AssertionError if assertions are enabled, or a regular exception otherwise:
assert false : errorMessage.toString();
throw new IllegalStateException(errorMessage.toString());
}
}
private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection<SystemIndexDescriptor> descriptors) {
Optional<Automaton> automaton = descriptors.stream()
.map(descriptor -> Regex.simpleMatchToAutomaton(descriptor.getIndexPattern()))
.reduce(Operations::union);
return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE));
}
/**
* Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed
* descriptors overlap with any of the other patterns. If any do, throws an exception.
*
* @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide.
* @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
*/
static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) {
List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet().stream()
.flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
.sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
.collect(Collectors.toList());
// This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
// automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states
// per pattern should be low as well. If these assumptions change, this might need to be reworked.
sourceDescriptorPair.forEach(descriptorToCheck -> {
List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
.filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
.filter(d -> overlaps(descriptorToCheck.v2(), d.v2()))
.collect(Collectors.toList());
if (descriptorsMatchingThisPattern.isEmpty() == false) {
throw new IllegalStateException("a system index descriptor [" + descriptorToCheck.v2() + "] from plugin [" +
descriptorToCheck.v1() + "] overlaps with other system index descriptors: [" +
descriptorsMatchingThisPattern.stream()
.map(descriptor -> descriptor.v2() + " from plugin [" + descriptor.v1() + "]")
.collect(Collectors.joining(", ")));
}
});
}
private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) {
Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern());
Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern());
return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false;
}
}

14
server/src/main/java/org/elasticsearch/node/Node.java
View File

@ -55,6 +55,7 @@ import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
import org.elasticsearch.cluster.metadata.MetadataIndexUpgradeService;
import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService;
import org.elasticsearch.cluster.metadata.TemplateUpgradeService;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@ -105,12 +106,14 @@ import org.elasticsearch.gateway.MetaStateService;
import org.elasticsearch.gateway.PersistedClusterStateService;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingPressure;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.engine.EngineFactory;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.breaker.BreakerSettings;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
@ -483,11 +486,7 @@ public class Node implements Closeable {
.collect(Collectors.toMap(
plugin -> plugin.getClass().getSimpleName(),
plugin -> plugin.getSystemIndexDescriptors(settings))));
SystemIndexDescriptor.checkForOverlappingPatterns(systemIndexDescriptorMap);
final List<SystemIndexDescriptor> systemIndexDescriptors = systemIndexDescriptorMap.values().stream()
.flatMap(Collection::stream)
.collect(Collectors.toList());
final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap);
final RerouteService rerouteService
= new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute);
@ -514,7 +513,7 @@ public class Node implements Closeable {
settingsModule.getIndexScopedSettings(),
threadPool,
xContentRegistry,
systemIndexDescriptors,
systemIndices,
forbidPrivateIndexSettings
);
@ -544,7 +543,8 @@ public class Node implements Closeable {
.collect(Collectors.toList());
final MetadataUpgrader metadataUpgrader = new MetadataUpgrader(indexTemplateMetadataUpgraders);
final MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService(settings, xContentRegistry,
indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings());
indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), systemIndices);
clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService));
new TemplateUpgradeService(client, clusterService, threadPool, indexTemplateMetadataUpgraders);
final Transport transport = networkModule.getTransportSupplier().get();
Set<String> taskHeaders = Stream.concat(

10
server/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -544,7 +544,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
private Executor getExecutor(IndexShard indexShard) {
assert indexShard != null;
return threadPool.executor(indexShard.indexSettings().isSearchThrottled() ? Names.SEARCH_THROTTLED : Names.SEARCH);
final String executorName;
if (indexShard.isSystem()) {
executorName = Names.SYSTEM_READ;
} else if (indexShard.indexSettings().isSearchThrottled()) {
executorName = Names.SEARCH_THROTTLED;
} else {
executorName = Names.SEARCH;
}
return threadPool.executor(executorName);
}
public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,

2
server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View File

@ -325,7 +325,7 @@ public class RestoreService implements ClusterStateApplier {
// Make sure that the index we are about to create has a validate name
boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(snapshotIndexMetadata.getSettings());
createIndexService.validateIndexName(renamedIndexName, currentState);
createIndexService.validateDotIndex(renamedIndexName, currentState, isHidden);
createIndexService.validateDotIndex(renamedIndexName, isHidden);
createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), false);
IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata)
.state(IndexMetadata.State.OPEN)

3
server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
View File

@ -81,6 +81,7 @@ public class ThreadPool implements ReportingService<ThreadPoolInfo>, Scheduler {
public static final String FORCE_MERGE = "force_merge";
public static final String FETCH_SHARD_STARTED = "fetch_shard_started";
public static final String FETCH_SHARD_STORE = "fetch_shard_store";
public static final String SYSTEM_READ = "system_read";
}
public enum ThreadPoolType {
@ -138,6 +139,7 @@ public class ThreadPool implements ReportingService<ThreadPoolInfo>, Scheduler {
map.put(Names.FETCH_SHARD_STARTED, ThreadPoolType.SCALING);
map.put(Names.FETCH_SHARD_STORE, ThreadPoolType.SCALING);
map.put(Names.SEARCH_THROTTLED, ThreadPoolType.FIXED_AUTO_QUEUE_SIZE);
map.put(Names.SYSTEM_READ, ThreadPoolType.FIXED);
THREAD_POOL_TYPES = Collections.unmodifiableMap(map);
}
@ -192,6 +194,7 @@ public class ThreadPool implements ReportingService<ThreadPoolInfo>, Scheduler {
builders.put(Names.FORCE_MERGE, new FixedExecutorBuilder(settings, Names.FORCE_MERGE, 1, -1));
builders.put(Names.FETCH_SHARD_STORE,
new ScalingExecutorBuilder(Names.FETCH_SHARD_STORE, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5)));
builders.put(Names.SYSTEM_READ, new FixedExecutorBuilder(settings, Names.SYSTEM_READ, halfProcMaxAt5, 2000, false));
for (final ExecutorBuilder<?> builder : customBuilders) {
if (builders.containsKey(builder.name())) {
throw new IllegalArgumentException("builder with name [" + builder.name() + "] already exists");

6
server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
View File

@ -121,7 +121,8 @@ public class ClusterRerouteResponseTests extends ESTestCase {
" \"in_sync_allocations\" : {\n" +
" \"0\" : [ ]\n" +
" },\n" +
" \"rollover_info\" : { }\n" +
" \"rollover_info\" : { },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -218,7 +219,8 @@ public class ClusterRerouteResponseTests extends ESTestCase {
" \"in_sync_allocations\" : {\n" +
" \"0\" : [ ]\n" +
" },\n" +
" \"rollover_info\" : { }\n" +
" \"rollover_info\" : { },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +

8
server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View File

@ -69,6 +69,7 @@ import org.elasticsearch.index.shard.IndexEventListener;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.TestThreadPool;
@ -82,6 +83,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -492,7 +494,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY,
clusterService, indicesService, allocationService, null, shardLimitValidator, env,
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, Collections.emptyList(), false);
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService,
new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService,
@ -585,7 +587,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY,
clusterService, indicesService, allocationService, null, shardLimitValidator, env,
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, Collections.emptyList(), false);
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService,
new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService,
@ -707,7 +709,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY,
clusterService, indicesService, allocationService, null, null, env,
null, testThreadPool, null, Collections.emptyList(), false);
null, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService,
new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService,

12
server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
View File

@ -244,7 +244,8 @@ public class ClusterStateTests extends ESTestCase {
" \"met_conditions\" : { },\n" +
" \"time\" : 1\n" +
" }\n" +
" }\n" +
" },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -427,7 +428,8 @@ public class ClusterStateTests extends ESTestCase {
" \"met_conditions\" : { },\n" +
" \"time\" : 1\n" +
" }\n" +
" }\n" +
" },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -619,7 +621,8 @@ public class ClusterStateTests extends ESTestCase {
" \"met_conditions\" : { },\n" +
" \"time\" : 1\n" +
" }\n" +
" }\n" +
" },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -745,7 +748,8 @@ public class ClusterStateTests extends ESTestCase {
" \"in_sync_allocations\" : {\n" +
" \"0\" : [ ]\n" +
" },\n" +
" \"rollover_info\" : { }\n" +
" \"rollover_info\" : { },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +

5
server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
View File

@ -79,6 +79,7 @@ public class IndexMetadataTests extends ESTestCase {
public void testIndexMetadataSerialization() throws IOException {
Integer numShard = randomFrom(1, 2, 4, 8, 16);
int numberOfReplicas = randomIntBetween(0, 10);
final boolean system = randomBoolean();
Map<String, String> customMap = new HashMap<>();
customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10));
customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15));
@ -91,6 +92,7 @@ public class IndexMetadataTests extends ESTestCase {
.creationDate(randomLong())
.primaryTerm(0, 2)
.setRoutingNumShards(32)
.system(system)
.putCustom("my_custom", customMap)
.putRolloverInfo(
new RolloverInfo(randomAlphaOfLength(5),
@ -98,6 +100,7 @@ public class IndexMetadataTests extends ESTestCase {
new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())),
new MaxDocsCondition(randomNonNegativeLong())),
randomNonNegativeLong())).build();
assertEquals(system, metadata.isSystem());
final XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
@ -116,6 +119,7 @@ public class IndexMetadataTests extends ESTestCase {
assertEquals(metadata.getCreationDate(), fromXContentMeta.getCreationDate());
assertEquals(metadata.getRoutingFactor(), fromXContentMeta.getRoutingFactor());
assertEquals(metadata.primaryTerm(0), fromXContentMeta.primaryTerm(0));
assertEquals(metadata.isSystem(), fromXContentMeta.isSystem());
ImmutableOpenMap.Builder<String, DiffableStringMap> expectedCustomBuilder = ImmutableOpenMap.builder();
expectedCustomBuilder.put("my_custom", new DiffableStringMap(customMap));
ImmutableOpenMap<String, DiffableStringMap> expectedCustom = expectedCustomBuilder.build();
@ -139,6 +143,7 @@ public class IndexMetadataTests extends ESTestCase {
assertEquals(metadata.getRolloverInfos(), deserialized.getRolloverInfos());
assertEquals(deserialized.getCustomData(), expectedCustom);
assertEquals(metadata.getCustomData(), deserialized.getCustomData());
assertEquals(metadata.isSystem(), deserialized.isSystem());
}
}

32
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
View File

@ -59,6 +59,7 @@ import org.elasticsearch.indices.InvalidAliasNameException;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
@ -107,7 +108,6 @@ import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.pars
import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases;
import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
@ -475,7 +475,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
null,
threadPool,
null,
Collections.emptyList(),
new SystemIndices(Collections.emptyMap()),
false
);
validateIndexName(checkerService, "index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
@ -541,7 +541,6 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
systemIndexDescriptors.add(new SystemIndexDescriptor(".test", "test"));
systemIndexDescriptors.add(new SystemIndexDescriptor(".test3", "test"));
systemIndexDescriptors.add(new SystemIndexDescriptor(".pattern-test*", "test-1"));
systemIndexDescriptors.add(new SystemIndexDescriptor(".pattern-test-overlapping", "test-2"));
withTemporaryClusterService(((clusterService, threadPool) -> {
MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
@ -554,33 +553,25 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
null,
threadPool,
null,
systemIndexDescriptors,
new SystemIndices(Collections.singletonMap("foo", systemIndexDescriptors)),
false
);
// Check deprecations
checkerService.validateDotIndex(".test2", ClusterState.EMPTY_STATE, false);
assertFalse(checkerService.validateDotIndex(".test2", false));
assertWarnings("index name [.test2] starts with a dot '.', in the next major version, index " +
"names starting with a dot are reserved for hidden indices and system indices");
// Check non-system hidden indices don't trigger a warning
checkerService.validateDotIndex(".test2", ClusterState.EMPTY_STATE, true);
assertFalse(checkerService.validateDotIndex(".test2", true));
// Check NO deprecation warnings if we give the index name
checkerService.validateDotIndex(".test", ClusterState.EMPTY_STATE, false);
checkerService.validateDotIndex(".test3", ClusterState.EMPTY_STATE, false);
assertTrue(checkerService.validateDotIndex(".test", false));
assertTrue(checkerService.validateDotIndex(".test3", false));
// Check that patterns with wildcards work
checkerService.validateDotIndex(".pattern-test", ClusterState.EMPTY_STATE, false);
checkerService.validateDotIndex(".pattern-test-with-suffix", ClusterState.EMPTY_STATE, false);
checkerService.validateDotIndex(".pattern-test-other-suffix", ClusterState.EMPTY_STATE, false);
// Check that an exception is thrown if more than one descriptor matches the index name
AssertionError exception = expectThrows(AssertionError.class,
() -> checkerService.validateDotIndex(".pattern-test-overlapping", ClusterState.EMPTY_STATE, false));
assertThat(exception.getMessage(),
containsString("index name [.pattern-test-overlapping] is claimed as a system index by multiple system index patterns:"));
assertThat(exception.getMessage(), containsString("pattern: [.pattern-test*], description: [test-1]"));
assertThat(exception.getMessage(), containsString("pattern: [.pattern-test-overlapping], description: [test-2]"));
assertTrue(checkerService.validateDotIndex(".pattern-test", false));
assertTrue(checkerService.validateDotIndex(".pattern-test-with-suffix", false));
assertTrue(checkerService.validateDotIndex(".pattern-test-other-suffix", false));
}));
}
@ -873,7 +864,8 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
.put(SETTING_NUMBER_OF_SHARDS, 1)
.build();
List<AliasMetadata> aliases = singletonList(AliasMetadata.builder("alias1").build());
IndexMetadata indexMetadata = buildIndexMetadata("test", aliases, () -> null, () -> null, indexSettings, 4, sourceIndexMetadata);
IndexMetadata indexMetadata =
buildIndexMetadata("test", aliases, () -> null, () -> null, indexSettings, 4, sourceIndexMetadata, false);
assertThat(indexMetadata.getSettings().getAsBoolean(INDEX_SOFT_DELETES_SETTING.getKey(), true), is(false));
assertThat(indexMetadata.getAliases().size(), is(1));

5
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
View File

@ -52,6 +52,7 @@ import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexTemplateException;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
@ -1428,7 +1429,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
null,
xContentRegistry,
Collections.emptyList(),
new SystemIndices(Collections.emptyMap()),
true
);
MetadataIndexTemplateService service = new MetadataIndexTemplateService(null, createIndexService,
@ -1485,7 +1486,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
null,
xContentRegistry(),
Collections.emptyList(),
new SystemIndices(Collections.emptyMap()),
true
);
return new MetadataIndexTemplateService(

26
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.cluster.metadata;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.test.ESTestCase;
@ -128,12 +130,34 @@ public class MetadataIndexUpgradeServiceTests extends ESTestCase {
service.upgradeIndexMetadata(goodMeta, Version.CURRENT.minimumIndexCompatibilityVersion());
}
public void testMaybeMarkAsSystemIndex() {
MetadataIndexUpgradeService service = getMetadataIndexUpgradeService();
IndexMetadata src = newIndexMeta("foo", Settings.EMPTY);
assertFalse(src.isSystem());
IndexMetadata indexMetadata = service.maybeMarkAsSystemIndex(src);
assertSame(indexMetadata, src);
src = newIndexMeta(".system", Settings.EMPTY);
assertFalse(src.isSystem());
indexMetadata = service.maybeMarkAsSystemIndex(src);
assertNotSame(indexMetadata, src);
assertTrue(indexMetadata.isSystem());
// test with the whole upgrade
assertFalse(src.isSystem());
indexMetadata = service.upgradeIndexMetadata(src, Version.CURRENT.minimumIndexCompatibilityVersion());
assertTrue(indexMetadata.isSystem());
}
private MetadataIndexUpgradeService getMetadataIndexUpgradeService() {
return new MetadataIndexUpgradeService(
Settings.EMPTY,
xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
new SystemIndices(Collections.singletonMap("system-plugin",
Collections.singletonList(new SystemIndexDescriptor(".system", "a system index"))))
);
}
public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {

9
server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
View File

@ -299,7 +299,8 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
" \"in_sync_allocations\" : {\n" +
" \"0\" : [ ]\n" +
" },\n" +
" \"rollover_info\" : { }\n" +
" \"rollover_info\" : { },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -461,7 +462,8 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
" \"met_conditions\" : { },\n" +
" \"time\" : 1\n" +
" }\n" +
" }\n" +
" },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +
@ -566,7 +568,8 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
" \"met_conditions\" : { },\n" +
" \"time\" : 1\n" +
" }\n" +
" }\n" +
" },\n" +
" \"system\" : false\n" +
" }\n" +
" },\n" +
" \"index-graveyard\" : {\n" +

2
server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
View File

@ -159,7 +159,7 @@ public class GatewayMetaStateTests extends ESTestCase {
private final boolean upgrade;
MockMetadataIndexUpgradeService(boolean upgrade) {
super(Settings.EMPTY, null, null, null);
super(Settings.EMPTY, null, null, null, null);
this.upgrade = upgrade;
}

53
server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java
View File

@ -21,13 +21,7 @@ package org.elasticsearch.indices;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;
public class SystemIndexDescriptorTests extends ESTestCase {
@ -67,51 +61,4 @@ public class SystemIndexDescriptorTests extends ESTestCase {
assertThat(ex.getMessage(), containsString("must not start with the character sequence [.*] to prevent conflicts"));
}
}
public void testBasicOverlappingPatterns() {
SystemIndexDescriptor broadPattern = new SystemIndexDescriptor(".a*c*", "test");
SystemIndexDescriptor notOverlapping = new SystemIndexDescriptor(".bbbddd*", "test");
SystemIndexDescriptor overlapping1 = new SystemIndexDescriptor(".ac*", "test");
SystemIndexDescriptor overlapping2 = new SystemIndexDescriptor(".aaaabbbccc", "test");
SystemIndexDescriptor overlapping3 = new SystemIndexDescriptor(".aaabb*cccddd*", "test");
// These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent
// across tests
String broadPatternSource = "AAA" + randomAlphaOfLength(5);
String otherSource = "ZZZ" + randomAlphaOfLength(6);
Map<String, Collection<SystemIndexDescriptor>> descriptors = new HashMap<>();
descriptors.put(broadPatternSource, Arrays.asList(broadPattern));
descriptors.put(otherSource, Arrays.asList(notOverlapping, overlapping1, overlapping2, overlapping3));
IllegalStateException exception = expectThrows(IllegalStateException.class,
() -> SystemIndexDescriptor.checkForOverlappingPatterns(descriptors));
assertThat(exception.getMessage(), containsString("a system index descriptor [" + broadPattern +
"] from plugin [" + broadPatternSource + "] overlaps with other system index descriptors:"));
String fromPluginString = " from plugin [" + otherSource + "]";
assertThat(exception.getMessage(), containsString(overlapping1.toString() + fromPluginString));
assertThat(exception.getMessage(), containsString(overlapping2.toString() + fromPluginString));
assertThat(exception.getMessage(), containsString(overlapping3.toString() + fromPluginString));
assertThat(exception.getMessage(), not(containsString(notOverlapping.toString())));
}
public void testComplexOverlappingPatterns() {
// These patterns are slightly more complex to detect because pattern1 does not match pattern2 and vice versa
SystemIndexDescriptor pattern1 = new SystemIndexDescriptor(".a*c", "test");
SystemIndexDescriptor pattern2 = new SystemIndexDescriptor(".ab*", "test");
// These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent
// across tests
String source1 = "AAA" + randomAlphaOfLength(5);
String source2 = "ZZZ" + randomAlphaOfLength(6);
Map<String, Collection<SystemIndexDescriptor>> descriptors = new HashMap<>();
descriptors.put(source1, Arrays.asList(pattern1));
descriptors.put(source2, Arrays.asList(pattern2));
IllegalStateException exception = expectThrows(IllegalStateException.class,
() -> SystemIndexDescriptor.checkForOverlappingPatterns(descriptors));
assertThat(exception.getMessage(), containsString("a system index descriptor [" + pattern1 +
"] from plugin [" + source1 + "] overlaps with other system index descriptors:"));
assertThat(exception.getMessage(), containsString(pattern2.toString() + " from plugin [" + source2 + "]"));
}
}

87
server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
public class SystemIndicesTests extends ESTestCase {
public void testBasicOverlappingPatterns() {
SystemIndexDescriptor broadPattern = new SystemIndexDescriptor(".a*c*", "test");
SystemIndexDescriptor notOverlapping = new SystemIndexDescriptor(".bbbddd*", "test");
SystemIndexDescriptor overlapping1 = new SystemIndexDescriptor(".ac*", "test");
SystemIndexDescriptor overlapping2 = new SystemIndexDescriptor(".aaaabbbccc", "test");
SystemIndexDescriptor overlapping3 = new SystemIndexDescriptor(".aaabb*cccddd*", "test");
// These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent
// across tests
String broadPatternSource = "AAA" + randomAlphaOfLength(5);
String otherSource = "ZZZ" + randomAlphaOfLength(6);
Map<String, Collection<SystemIndexDescriptor>> descriptors = new HashMap<>();
descriptors.put(broadPatternSource, Collections.singletonList(broadPattern));
descriptors.put(otherSource, Arrays.asList(notOverlapping, overlapping1, overlapping2, overlapping3));
IllegalStateException exception = expectThrows(IllegalStateException.class,
() -> SystemIndices.checkForOverlappingPatterns(descriptors));
assertThat(exception.getMessage(), containsString("a system index descriptor [" + broadPattern +
"] from plugin [" + broadPatternSource + "] overlaps with other system index descriptors:"));
String fromPluginString = " from plugin [" + otherSource + "]";
assertThat(exception.getMessage(), containsString(overlapping1.toString() + fromPluginString));
assertThat(exception.getMessage(), containsString(overlapping2.toString() + fromPluginString));
assertThat(exception.getMessage(), containsString(overlapping3.toString() + fromPluginString));
assertThat(exception.getMessage(), not(containsString(notOverlapping.toString())));
IllegalStateException constructorException = expectThrows(IllegalStateException.class, () -> new SystemIndices(descriptors));
assertThat(constructorException.getMessage(), equalTo(exception.getMessage()));
}
public void testComplexOverlappingPatterns() {
// These patterns are slightly more complex to detect because pattern1 does not match pattern2 and vice versa
SystemIndexDescriptor pattern1 = new SystemIndexDescriptor(".a*c", "test");
SystemIndexDescriptor pattern2 = new SystemIndexDescriptor(".ab*", "test");
// These sources have fixed prefixes to make sure they sort in the same order, so that the error message is consistent
// across tests
String source1 = "AAA" + randomAlphaOfLength(5);
String source2 = "ZZZ" + randomAlphaOfLength(6);
Map<String, Collection<SystemIndexDescriptor>> descriptors = new HashMap<>();
descriptors.put(source1, Collections.singletonList(pattern1));
descriptors.put(source2, Collections.singletonList(pattern2));
IllegalStateException exception = expectThrows(IllegalStateException.class,
() -> SystemIndices.checkForOverlappingPatterns(descriptors));
assertThat(exception.getMessage(), containsString("a system index descriptor [" + pattern1 +
"] from plugin [" + source1 + "] overlaps with other system index descriptors:"));
assertThat(exception.getMessage(), containsString(pattern2.toString() + " from plugin [" + source2 + "]"));
IllegalStateException constructorException = expectThrows(IllegalStateException.class, () -> new SystemIndices(descriptors));
assertThat(constructorException.getMessage(), equalTo(exception.getMessage()));
}
}

7
server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
View File

@ -90,6 +90,7 @@ import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.IndexEventListener;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.test.gateway.TestGatewayAllocator;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.Transport;
@ -106,6 +107,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.env.Environment.PATH_HOME_SETTING;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.Assert.assertThat;
@ -189,7 +191,8 @@ public class ClusterStateChanges {
TransportService.NOOP_TRANSPORT_INTERCEPTOR,
boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings,
Collections.emptySet());
MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService(SETTINGS, xContentRegistry, null, null) {
MetadataIndexUpgradeService metadataIndexUpgradeService =
new MetadataIndexUpgradeService(SETTINGS, xContentRegistry, null, null, null) {
// metadata upgrader should do nothing
@Override
public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) {
@ -210,7 +213,7 @@ public class ClusterStateChanges {
allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, shardLimitValidator, threadPool);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(SETTINGS, clusterService, indicesService,
allocationService, new AliasValidator(), shardLimitValidator, environment,
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, Collections.emptyList(), true);
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, new SystemIndices(emptyMap()), true);
transportCloseIndexAction = new TransportCloseIndexAction(SETTINGS, transportService, clusterService, threadPool,
indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations);

6
server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
View File

@ -159,6 +159,7 @@ import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.ShardLimitValidator;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
@ -1565,7 +1566,7 @@ public class SnapshotResiliencyTests extends ESTestCase {
final MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService(settings, clusterService,
indicesService,
allocationService, new AliasValidator(), shardLimitValidator, environment, indexScopedSettings,
threadPool, namedXContentRegistry, Collections.emptyList(), false);
threadPool, namedXContentRegistry, new SystemIndices(emptyMap()), false);
actions.put(CreateIndexAction.INSTANCE,
new TransportCreateIndexAction(
transportService, clusterService, threadPool,
@ -1592,7 +1593,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
new MetadataIndexUpgradeService(
settings, namedXContentRegistry,
mapperRegistry,
indexScopedSettings),
indexScopedSettings,
new SystemIndices(emptyMap())),
clusterSettings,
shardLimitValidator
);