honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

System index auto-creation should not be disabled by user settings (#62984) (#63147) 

* Add System Indices check to AutoCreateIndex

By default, Elasticsearch auto-creates indices when a document is
submitted to a non-existent index. There is a setting that allows users
to disable this behavior. However, this setting should not apply to
system indices, so that Elasticsearch modules and plugins are able to
use auto-create behavior whether or not it is exposed to users.

This commit constructs the AutoCreateIndex object with a reference to
the SystemIndices object so that we bypass the check for the user-facing
autocreate setting when it's a system index that is being autocreated.

We also modify the logic in TransportBulkAction to make sure that if a
system index is included in a bulk request, we don't skip the
autocreation step.
This commit is contained in:
William Brafford 2020-10-01 16:26:07 -04:00 committed by GitHub
parent fc13b72cea
commit 6899ce6309
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 105 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
View File

@ -35,8 +35,11 @@ import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.test.ESTestCase;
import java.util.HashMap;
import static java.util.Collections.emptyMap;
import static org.hamcrest.Matchers.containsString;
@ -60,7 +63,8 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
.put(index("source2", "source_multi"), true)).build();
private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver();
private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER);
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER,
new SystemIndices(new HashMap<>()));
private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }");

7
server/src/main/java/org/elasticsearch/action/ActionModule.java
View File

@ -250,6 +250,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.index.seqno.RetentionLeaseActions;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.persistent.CompletionPersistentTaskAction;
import org.elasticsearch.persistent.RemovePersistentTaskAction;
@ -427,7 +428,7 @@ public class ActionModule extends AbstractModule {
public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
CircuitBreakerService circuitBreakerService, UsageService usageService) {
CircuitBreakerService circuitBreakerService, UsageService usageService, SystemIndices systemIndices) {
this.transportClient = transportClient;
this.settings = settings;
this.indexNameExpressionResolver = indexNameExpressionResolver;
@ -438,7 +439,9 @@ public class ActionModule extends AbstractModule {
this.threadPool = threadPool;
actions = setupActions(actionPlugins);
actionFilters = setupActionFilters(actionPlugins);
autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
autoCreateIndex = transportClient
? null
: new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, systemIndices);
destructiveOperations = new DestructiveOperations(settings, clusterSettings);
Set<RestHeaderDefinition> headers = Stream.concat(
actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()),

27
server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View File

@ -229,7 +229,9 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
return;
}
if (needToCheck()) {
final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
if (includesSystem || needToCheck()) {
// Attempt to create all the indices that we're going to need during the bulk before we start.
// Step 1: collect all the indices in the request
final Map<String, Boolean> indices = bulkRequest.requests.stream()
@ -352,15 +354,20 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
}
boolean isOnlySystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
final boolean onlySystem = request.getIndices().stream().allMatch(indexName -> {
final IndexAbstraction abstraction = indicesLookup.get(indexName);
if (abstraction != null) {
return abstraction.isSystem();
} else {
return systemIndices.isSystemIndex(indexName);
}
});
return onlySystem;
return request.getIndices().stream().allMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
}
boolean includesSystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
return request.getIndices().stream().anyMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
}
private boolean isSystemIndex(SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices, String indexName) {
final IndexAbstraction abstraction = indicesLookup.get(indexName);
if (abstraction != null) {
return abstraction.isSystem();
} else {
return systemIndices.isSystemIndex(indexName);
}
}
boolean needToCheck() {

14
server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.SystemIndices;
import java.util.ArrayList;
import java.util.List;
@ -46,11 +47,16 @@ public final class AutoCreateIndex {
private final boolean dynamicMappingDisabled;
private final IndexNameExpressionResolver resolver;
private final SystemIndices systemIndices;
private volatile AutoCreate autoCreate;
public AutoCreateIndex(Settings settings, ClusterSettings clusterSettings, IndexNameExpressionResolver resolver) {
public AutoCreateIndex(Settings settings,
ClusterSettings clusterSettings,
IndexNameExpressionResolver resolver,
SystemIndices systemIndices) {
this.resolver = resolver;
dynamicMappingDisabled = !MapperService.INDEX_MAPPER_DYNAMIC_SETTING.get(settings);
this.systemIndices = systemIndices;
this.autoCreate = AUTO_CREATE_INDEX_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate);
}
@ -70,6 +76,12 @@ public final class AutoCreateIndex {
if (resolver.hasIndexAbstraction(index, state)) {
return false;
}
// Always auto-create system indexes
if (systemIndices.isSystemIndex(index)) {
return true;
}
// One volatile read, so that all checks are done against the same instance:
final AutoCreate autoCreate = this.autoCreate;
if (autoCreate.autoCreateIndex == false) {

20
server/src/main/java/org/elasticsearch/client/transport/TransportClient.java
View File

@ -48,6 +48,7 @@ import org.elasticsearch.common.util.PageCacheRecycler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.Node;
@ -67,7 +68,6 @@ import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -75,6 +75,10 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableList;
import static java.util.stream.Collectors.toList;
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
@ -150,7 +154,7 @@ public abstract class TransportClient extends AbstractClient {
final List<Closeable> resourcesToClose = new ArrayList<>();
final ThreadPool threadPool = new ThreadPool(settings);
resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
final NetworkService networkService = new NetworkService(Collections.emptyList());
final NetworkService networkService = new NetworkService(emptyList());
try {
final List<Setting<?>> additionalSettings = new ArrayList<>(pluginsService.getPluginSettings());
final List<String> additionalSettingsFilter = new ArrayList<>(pluginsService.getPluginSettingsFilter());
@ -158,10 +162,10 @@ public abstract class TransportClient extends AbstractClient {
additionalSettings.addAll(builder.getRegisteredSettings());
}
SettingsModule settingsModule =
new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet());
new SettingsModule(settings, additionalSettings, additionalSettingsFilter, emptySet());
SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
IndicesModule indicesModule = new IndicesModule(emptyList());
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.addAll(NetworkModule.getNamedWriteables());
entries.addAll(searchModule.getNamedWriteables());
@ -185,11 +189,11 @@ public abstract class TransportClient extends AbstractClient {
modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(),
settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool,
pluginsService.filterPlugins(ActionPlugin.class), null, null, null);
pluginsService.filterPlugins(ActionPlugin.class), null, null, null, new SystemIndices(emptyMap()));
modules.add(actionModule);
CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
Collections.emptyList(),
emptyList(),
settingsModule.getClusterSettings());
resourcesToClose.add(circuitBreakerService);
PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings);
@ -202,7 +206,7 @@ public abstract class TransportClient extends AbstractClient {
final TransportService transportService = new TransportService(settings, transport, threadPool,
networkModule.getTransportInterceptor(),
boundTransportAddress -> DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 0),
UUIDs.randomBase64UUID()), null, Collections.emptySet());
UUIDs.randomBase64UUID()), null, emptySet());
modules.add((b -> {
b.bind(BigArrays.class).toInstance(bigArrays);
b.bind(PageCacheRecycler.class).toInstance(pageCacheRecycler);
@ -300,7 +304,7 @@ public abstract class TransportClient extends AbstractClient {
private TransportClient(ClientTemplate template) {
super(template.getSettings(), template.getThreadPool());
this.injector = template.injector;
this.pluginLifecycleComponents = Collections.unmodifiableList(template.pluginLifecycleComponents);
this.pluginLifecycleComponents = unmodifiableList(template.pluginLifecycleComponents);
this.nodesService = template.nodesService;
this.proxy = template.proxy;
this.namedWriteableRegistry = template.namedWriteableRegistry;

2
server/src/main/java/org/elasticsearch/node/Node.java
View File

@ -539,7 +539,7 @@ public class Node implements Closeable {
ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService, systemIndices);
modules.add(actionModule);
final RestController restController = actionModule.getRestController();

6
server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
View File

@ -109,7 +109,7 @@ public class ActionModuleTests extends ESTestCase {
UsageService usageService = new UsageService();
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
null, usageService);
null, usageService, null);
actionModule.initRestHandlers(null);
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
Exception e = expectThrows(IllegalArgumentException.class, () ->
@ -148,7 +148,7 @@ public class ActionModuleTests extends ESTestCase {
UsageService usageService = new UsageService();
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
singletonList(dupsMainAction), null, null, usageService);
singletonList(dupsMainAction), null, null, usageService, null);
Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
} finally {
@ -182,7 +182,7 @@ public class ActionModuleTests extends ESTestCase {
UsageService usageService = new UsageService();
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
singletonList(registersFakeHandler), null, null, usageService);
singletonList(registersFakeHandler), null, null, usageService, null);
actionModule.initRestHandlers(null);
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
Exception e = expectThrows(IllegalArgumentException.class, () ->

3
server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
View File

@ -147,7 +147,8 @@ public class TransportBulkActionIngestTests extends ESTestCase {
null, null, new ActionFilters(Collections.emptySet()), null,
new AutoCreateIndex(
SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
new IndexNameExpressionResolver()
new IndexNameExpressionResolver(),
new SystemIndices(emptyMap())
), new IndexingPressure(SETTINGS), new SystemIndices(emptyMap())
);
}

24
server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
View File

@ -88,7 +88,7 @@ public class TransportBulkActionTests extends ESTestCase {
TestTransportBulkAction() {
super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null,
null, new ActionFilters(Collections.emptySet()), new Resolver(),
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()),
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(emptyMap())),
new IndexingPressure(Settings.EMPTY), new SystemIndices(emptyMap()));
}
@ -271,6 +271,28 @@ public class TransportBulkActionTests extends ESTestCase {
assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
}
public void testIncludesSystem() {
SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build();
indicesLookup.put(".foo",
new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
indicesLookup.put(".bar",
new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin",
org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(".test", ""))));
List<String> onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar");
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar", ".test");
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
List<String> nonSystem = org.elasticsearch.common.collect.List.of("foo", "bar");
assertFalse(bulkAction.includesSystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices));
List<String> mixed = org.elasticsearch.common.collect.List.of(".foo", ".test", "other");
assertTrue(bulkAction.includesSystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
}
private BulkRequest buildBulkRequest(List<String> indices) {
BulkRequest request = new BulkRequest();
for (String index : indices) {

24
server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
View File

@ -29,6 +29,8 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.test.ESTestCase;
import java.util.HashMap;
@ -39,6 +41,8 @@ import static org.hamcrest.CoreMatchers.equalTo;
public class AutoCreateIndexTests extends ESTestCase {
private static final String TEST_SYSTEM_INDEX_NAME = ".test-system-index";
public void testParseFailed() {
try {
Settings settings = Settings.builder().put("action.auto_create_index", ",,,").build();
@ -89,6 +93,12 @@ public class AutoCreateIndexTests extends ESTestCase {
assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage());
}
public void testSystemIndexWithAutoCreationDisabled() {
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build();
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
assertThat(autoCreateIndex.shouldAutoCreate(TEST_SYSTEM_INDEX_NAME, buildClusterState()), equalTo(true));
}
public void testAutoCreationEnabled() {
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true).build();
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
@ -127,6 +137,13 @@ public class AutoCreateIndexTests extends ESTestCase {
expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
}
public void testAutoCreationSystemIndexPatternDisabled() {
Settings settings =
Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), "-" + TEST_SYSTEM_INDEX_NAME + "*").build();
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
assertThat(autoCreateIndex.shouldAutoCreate(TEST_SYSTEM_INDEX_NAME, buildClusterState()), equalTo(true));
}
public void testAutoCreationMultiplePatternsWithWildcards() {
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(),
randomFrom("+test*,-index*", "test*,-index*")).build();
@ -177,7 +194,8 @@ public class AutoCreateIndexTests extends ESTestCase {
ClusterSettings clusterSettings = new ClusterSettings(settings,
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, new IndexNameExpressionResolver());
AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, new IndexNameExpressionResolver(),
new SystemIndices(org.elasticsearch.common.collect.Map.of()));
assertThat(autoCreateIndex.getAutoCreate().isAutoCreateIndex(), equalTo(value));
Settings newSettings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), !value).build();
@ -201,8 +219,10 @@ public class AutoCreateIndexTests extends ESTestCase {
}
private AutoCreateIndex newAutoCreateIndex(Settings settings) {
SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin",
org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, ""))));
return new AutoCreateIndex(settings, new ClusterSettings(settings,
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver());
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(), systemIndices);
}
private void expectNotMatch(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index) {

3
server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
View File

@ -1587,7 +1587,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(),
Collections.emptyList(), client),
transportShardBulkAction, client, actionFilters, indexNameExpressionResolver,
new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver), new IndexingPressure(settings),
new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(emptyMap())),
new IndexingPressure(settings),
new SystemIndices(emptyMap())
));
final RestoreService restoreService = new RestoreService(