Remove expansion of empty index arguments in RoutingTable

RoutingTables activePrimaryShardsGrouped(), allActiveShardsGrouped() and
allAssignedShardsGrouped() methods treated empty index array input
parameters as meaning "all" indices and expanded to the routing maps
keyset. However, the expansion of index names is now already done in
MetaData#concreteIndices(). Returning an empty index name list here
when a wildcard pattern didn't match any index name could lead to
problems like #9081 because the RoutingTable still expanded this
list of names to "_all". In case of e.g. the recovery endpoint this
could lead to problems.

Closes #9081
Closes #10148
This commit is contained in:
Christoph Büscher 2015-04-02 11:10:38 +02:00
parent d30d3b967e
commit 2beda3953d
6 changed files with 381 additions and 56 deletions

View File

@ -37,3 +37,44 @@
- gte: { test_1.shards.0.translog.total_time_in_millis: 0 }
- gte: { test_1.shards.0.start.check_index_time_in_millis: 0 }
- gte: { test_1.shards.0.start.total_time_in_millis: 0 }
---
"Indices recovery test index name not matching":
- do:
indices.create:
index: test_1
body:
settings:
index:
number_of_replicas: 0
- do:
cluster.health:
wait_for_status: green
- do:
catch: missing
indices.recovery:
index: [foobar]
---
"Indices recovery test, wildcard not matching any index":
- do:
indices.create:
index: test_1
body:
settings:
index:
number_of_replicas: 0
- do:
cluster.health:
wait_for_status: green
- do:
indices.recovery:
index: [v*]
- match: { $body: {} }

View File

@ -0,0 +1,58 @@
---
setup:
- do:
indices.create:
index: test_1
body:
settings:
index:
number_of_replicas: 0
number_of_shards: 5
- do:
indices.create:
index: test_2
body:
settings:
index:
number_of_replicas: 0
number_of_shards: 5
- do:
cluster.health:
wait_for_status: green
---
"Indices refresh test _all":
- do:
indices.refresh:
index: [_all]
- match: { _shards.total: 10 }
- match: { _shards.successful: 10 }
- match: { _shards.failed: 0 }
---
"Indices refresh test empty array":
- do:
indices.refresh:
index: []
- match: { _shards.total: 10 }
- match: { _shards.successful: 10 }
- match: { _shards.failed: 0 }
---
"Indices refresh test no-match wildcard":
- do:
indices.refresh:
index: [bla*]
- match: { _shards.total: 0 }
- match: { _shards.successful: 0 }
- match: { _shards.failed: 0 }

View File

@ -60,6 +60,16 @@ setup:
- is_true: indices.test1
- is_true: indices.test2
---
"Index - star, no match":
- do:
indices.stats: { index: 'bla*' }
- match: { _shards.total: 0 }
- is_true: _all
- is_false: indices.test1
- is_false: indices.test2
---
"Index - one index":
- do:

View File

@ -117,18 +117,29 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
}
/**
* All the shards (replicas) for the provided indices.
* All the shards (replicas) for all indices in this routing table.
*
* @param indices The indices to return all the shards (replicas), can be <tt>null</tt> or empty array to indicate all indices
* @return All the shards matching the specific index
* @throws IndexMissingException If an index passed does not exists
* @return All the shards
*/
public List<ShardRouting> allShards(String... indices) throws IndexMissingException {
public List<ShardRouting> allShards() throws IndexMissingException {
List<ShardRouting> shards = Lists.newArrayList();
if (indices == null || indices.length == 0) {
indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
}
String[] indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
for (String index : indices) {
List<ShardRouting> allShardsIndex = allShards(index);
shards.addAll(allShardsIndex);
}
return shards;
}
/**
* All the shards (replicas) for the provided index.
*
* @param index The index to return all the shards (replicas).
* @return All the shards matching the specific index
* @throws IndexMissingException If the index passed does not exists
*/
public List<ShardRouting> allShards(String index) throws IndexMissingException {
List<ShardRouting> shards = Lists.newArrayList();
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {
throw new IndexMissingException(new Index(index));
@ -138,42 +149,9 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
shards.add(shardRouting);
}
}
}
return shards;
}
/**
* All the shards (primary + replicas) for the provided indices grouped (each group is a single element, consisting
* of the shard). This is handy for components that expect to get group iterators, but still want in some
* cases to iterate over all the shards (and not just one shard in replication group).
*
* @param indices The indices to return all the shards (replicas), can be <tt>null</tt> or empty array to indicate all indices
* @return All the shards grouped into a single shard element group each
* @throws IndexMissingException If an index passed does not exists
* @see IndexRoutingTable#groupByAllIt()
*/
public GroupShardsIterator allShardsGrouped(String... indices) throws IndexMissingException {
// use list here since we need to maintain identity across shards
ArrayList<ShardIterator> set = new ArrayList<>();
if (indices == null || indices.length == 0) {
indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
}
for (String index : indices) {
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {
continue;
// we simply ignore indices that don't exists (make sense for operations that use it currently)
// throw new IndexMissingException(new Index(index));
}
for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
for (ShardRouting shardRouting : indexShardRoutingTable) {
set.add(shardRouting.shardsIt());
}
}
}
return new GroupShardsIterator(set);
}
public GroupShardsIterator allActiveShardsGrouped(String[] indices, boolean includeEmpty) throws IndexMissingException {
return allActiveShardsGrouped(indices, includeEmpty, false);
}
@ -188,15 +166,11 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
public GroupShardsIterator allActiveShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets) throws IndexMissingException {
// use list here since we need to maintain identity across shards
ArrayList<ShardIterator> set = new ArrayList<>();
if (indices == null || indices.length == 0) {
indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
}
for (String index : indices) {
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {
continue;
// we simply ignore indices that don't exists (make sense for operations that use it currently)
// throw new IndexMissingException(new Index(index));
}
for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
for (ShardRouting shardRouting : indexShardRoutingTable) {
@ -228,15 +202,11 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
public GroupShardsIterator allAssignedShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets) throws IndexMissingException {
// use list here since we need to maintain identity across shards
ArrayList<ShardIterator> set = new ArrayList<>();
if (indices == null || indices.length == 0) {
indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
}
for (String index : indices) {
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {
continue;
// we simply ignore indices that don't exists (make sense for operations that use it currently)
// throw new IndexMissingException(new Index(index));
}
for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
for (ShardRouting shardRouting : indexShardRoutingTable) {
@ -259,7 +229,7 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
* of the primary shard). This is handy for components that expect to get group iterators, but still want in some
* cases to iterate over all primary shards (and not just one shard in replication group).
*
* @param indices The indices to return all the shards (replicas), can be <tt>null</tt> or empty array to indicate all indices
* @param indices The indices to return all the shards (replicas)
* @return All the primary shards grouped into a single shard element group each
* @throws IndexMissingException If an index passed does not exists
* @see IndexRoutingTable#groupByAllIt()
@ -267,9 +237,6 @@ public class RoutingTable implements Iterable<IndexRoutingTable> {
public GroupShardsIterator activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) throws IndexMissingException {
// use list here since we need to maintain identity across shards
ArrayList<ShardIterator> set = new ArrayList<>();
if (indices == null || indices.length == 0) {
indices = indicesRouting.keySet().toArray(new String[indicesRouting.keySet().size()]);
}
for (String index : indices) {
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {

View File

@ -52,6 +52,7 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
import org.elasticsearch.action.bulk.BulkAction;
@ -92,6 +93,7 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.cluster.settings.ClusterDynamicSettings;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
@ -892,8 +894,7 @@ public class IndicesRequestTests extends ElasticsearchIntegrationTest {
private final Map<String, List<TransportRequest>> requests = new HashMap<>();
@Inject
public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool,
NodeSettingsService nodeSettingsService, @ClusterDynamicSettings DynamicSettings dynamicSettings) {
public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
super(settings, transport, threadPool);
}

View File

@ -0,0 +1,248 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.routing;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.test.ElasticsearchAllocationTestCase;
import org.junit.Before;
import org.junit.Test;
import static org.hamcrest.Matchers.nullValue;
import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.Matchers.is;
public class RoutingTableTest extends ElasticsearchAllocationTestCase {
private static final String TEST_INDEX_1 = "test1";
private static final String TEST_INDEX_2 = "test2";
private RoutingTable emptyRoutingTable;
private RoutingTable testRoutingTable;
private int numberOfShards;
private int numberOfReplicas;
private int shardsPerIndex;
private int totalNumberOfShards;
private final static Settings DEFAULT_SETTINGS = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
private final AllocationService ALLOCATION_SERVICE = createAllocationService(settingsBuilder()
.put("cluster.routing.allocation.concurrent_recoveries", 10)
.put("cluster.routing.allocation.node_initial_primaries_recoveries", 10)
.build());
private ClusterState clusterState;
@Before
public void setUp() throws Exception {
super.setUp();
this.numberOfShards = randomIntBetween(1, 5);
this.numberOfReplicas = randomIntBetween(1, 5);
this.shardsPerIndex = this.numberOfShards * (this.numberOfReplicas + 1);
this.totalNumberOfShards = this.shardsPerIndex * 2;
logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas.");
this.emptyRoutingTable = new RoutingTable.Builder().build();
MetaData metaData = MetaData.builder()
.put(createIndexMetaData(TEST_INDEX_1))
.put(createIndexMetaData(TEST_INDEX_2))
.build();
this.testRoutingTable = new RoutingTable.Builder()
.add(new IndexRoutingTable.Builder(TEST_INDEX_1).initializeAsNew(metaData.index(TEST_INDEX_1)).build())
.add(new IndexRoutingTable.Builder(TEST_INDEX_2).initializeAsNew(metaData.index(TEST_INDEX_2)).build())
.build();
this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(testRoutingTable).build();
}
/**
* puts primary shard routings into initializing state
*/
private void initPrimaries() {
logger.info("adding " + (this.numberOfReplicas + 1) + " nodes and performing rerouting");
Builder discoBuilder = DiscoveryNodes.builder();
for (int i=0; i<this.numberOfReplicas+1;i++) {
discoBuilder = discoBuilder.put(newNode("node"+i));
}
this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build();
RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState);
this.testRoutingTable = rerouteResult.routingTable();
assertThat(rerouteResult.changed(), is(true));
this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
}
private void startInitializingShards(String index) {
this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
logger.info("start primary shards for index " + index);
RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.routingNodes().shardsWithState(index, INITIALIZING));
this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
this.testRoutingTable = rerouteResult.routingTable();
}
private IndexMetaData.Builder createIndexMetaData(String indexName) {
return new IndexMetaData.Builder(indexName)
.settings(DEFAULT_SETTINGS)
.numberOfReplicas(this.numberOfReplicas)
.numberOfShards(this.numberOfShards);
}
@Test
public void testAllShards() {
assertThat(this.emptyRoutingTable.allShards().size(), is(0));
assertThat(this.testRoutingTable.allShards().size(), is(this.totalNumberOfShards));
assertThat(this.testRoutingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex));
try {
assertThat(this.testRoutingTable.allShards("not_existing").size(), is(0));
fail("Exception expected when calling allShards() with non existing index name");
} catch (IndexMissingException e) {
// expected
}
}
@Test
public void testHasIndex() {
assertThat(this.testRoutingTable.hasIndex(TEST_INDEX_1), is(true));
assertThat(this.testRoutingTable.hasIndex("foobar"), is(false));
}
@Test
public void testIndex() {
assertThat(this.testRoutingTable.index(TEST_INDEX_1).getIndex(), is(TEST_INDEX_1));
assertThat(this.testRoutingTable.index("foobar"), is(nullValue()));
}
@Test
public void testIndicesRouting() {
assertThat(this.testRoutingTable.indicesRouting().size(), is(2));
assertThat(this.testRoutingTable.getIndicesRouting().size(), is(2));
assertSame(this.testRoutingTable.getIndicesRouting(), this.testRoutingTable.indicesRouting());
}
@Test
public void testShardsWithState() {
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards));
initPrimaries();
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - 2 * this.numberOfShards));
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards));
startInitializingShards(TEST_INDEX_1);
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.numberOfShards));
int initializingExpected = this.numberOfShards + this.numberOfShards * this.numberOfReplicas;
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - this.numberOfShards));
startInitializingShards(TEST_INDEX_2);
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(2 * this.numberOfShards));
initializingExpected = 2 * this.numberOfShards * this.numberOfReplicas;
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected));
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards - initializingExpected - 2 * this.numberOfShards));
// now start all replicas too
startInitializingShards(TEST_INDEX_1);
startInitializingShards(TEST_INDEX_2);
assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.totalNumberOfShards));
}
@Test
public void testActivePrimaryShardsGrouped() {
assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0));
assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(0));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.numberOfShards));
initPrimaries();
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(0));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.numberOfShards));
startInitializingShards(TEST_INDEX_1);
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.numberOfShards));
startInitializingShards(TEST_INDEX_2);
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_2}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
assertThat(this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(2 * this.numberOfShards));
try {
this.testRoutingTable.activePrimaryShardsGrouped(new String[] {TEST_INDEX_1, "not_exists"}, true);
fail("Calling with non-existing index name should raise IndexMissingException");
} catch (IndexMissingException e) {
// expected
}
}
@Test
public void testAllActiveShardsGrouped() {
assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], true).size(), is(0));
assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], false).size(), is(0));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(0));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
initPrimaries();
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(0));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
startInitializingShards(TEST_INDEX_1);
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
startInitializingShards(TEST_INDEX_2);
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_2}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
assertThat(this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
try {
this.testRoutingTable.allActiveShardsGrouped(new String[] {TEST_INDEX_1, "not_exists"}, true);
} catch (IndexMissingException e) {
fail("Calling with non-existing index should be ignored at the moment");
}
}
@Test
public void testAllAssignedShardsGrouped() {
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(0));
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
initPrimaries();
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1}, false).size(), is(this.numberOfShards));
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1}, true).size(), is(this.shardsPerIndex));
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, false).size(), is(2 * this.numberOfShards));
assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1, TEST_INDEX_2}, true).size(), is(this.totalNumberOfShards));
try {
this.testRoutingTable.allAssignedShardsGrouped(new String[] {TEST_INDEX_1, "not_exists"}, false);
} catch (IndexMissingException e) {
fail("Calling with non-existing index should be ignored at the moment");
}
}
}