honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automatically prepare indices for splitting (#27451) 

Today we require users to prepare their indices for split operations.
Yet, we can do this automatically when an index is created which would
make the split feature a much more appealing option since it doesn't have
any 3rd party prerequisites anymore.

This change automatically sets the number of routinng shards such that
an index is guaranteed to be able to split once into twice as many shards.
The number of routing shards is scaled towards the default shard limit per index
such that indices with a smaller amount of shards can be split more often than
larger ones. For instance an index with 1 or 2 shards can be split 10x
(until it approaches 1024 shards) while an index created with 128 shards can only
be split 3x by a factor of 2. Please note this is just a default value and users
can still prepare their indices with `index.number_of_routing_shards` for custom
splitting.

NOTE: this change has an impact on the document distribution since we are changing
the hash space. Documents are still uniformly distributed across all shards but since
we are artificually changing the number of buckets in the consistent hashign space
document might be hashed into different shards compared to previous versions.

This is a 7.0 only change.
This commit is contained in:
Simon Willnauer 2017-11-23 09:48:54 +01:00 committed by GitHub
parent 05998f91d0
commit fadbe0de08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 507 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
View File

@ -264,7 +264,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
assertEquals(5, matrixStats.getFieldCount("num"));
assertEquals(56d, matrixStats.getMean("num"), 0d);
assertEquals(1830d, matrixStats.getVariance("num"), 0d);
assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d);
assertEquals(0.09340198804973046, matrixStats.getSkewness("num"), 0d);
assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d);
assertEquals(5, matrixStats.getFieldCount("num2"));
assertEquals(29d, matrixStats.getMean("num2"), 0d);

5
core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
View File

@ -163,10 +163,15 @@ public class TransportResizeAction extends TransportMasterNodeAction<ResizeReque
if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) {
throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index");
}
if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) {
// if we have a source index with 1 shards it's legal to set this
final boolean splitFromSingleShards = resizeRequest.getResizeType() == ResizeType.SPLIT && metaData.getNumberOfShards() == 1;
if (splitFromSingleShards == false) {
throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize");
}
}
String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index";
targetIndex.cause(cause);
Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);

18
core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -1333,25 +1333,33 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
* @return a the source shard ID to split off from
*/
public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) {
int numSourceShards = sourceIndexMetadata.getNumberOfShards();
if (shardId >= numTargetShards) {
throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: "
+ shardId);
}
int numSourceShards = sourceIndexMetadata.getNumberOfShards();
final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards);
assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata);
return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor);
}
private static void assertSplitMetadata(int numSourceShards, int numTargetShards, IndexMetaData sourceIndexMetadata) {
if (numSourceShards > numTargetShards) {
throw new IllegalArgumentException("the number of source shards [" + numSourceShards
+ "] must be less that the number of target shards [" + numTargetShards + "]");
}
int routingFactor = getRoutingFactor(numSourceShards, numTargetShards);
// now we verify that the numRoutingShards is valid in the source index
int routingNumShards = sourceIndexMetadata.getRoutingNumShards();
// note: if the number of shards is 1 in the source index we can just assume it's correct since from 1 we can split into anything
// this is important to special case here since we use this to validate this in various places in the code but allow to split form
// 1 to N but we never modify the sourceIndexMetadata to accommodate for that
int routingNumShards = numSourceShards == 1 ? numTargetShards : sourceIndexMetadata.getRoutingNumShards();
if (routingNumShards % numTargetShards != 0) {
throw new IllegalStateException("the number of routing shards ["
+ routingNumShards + "] must be a multiple of the target shards [" + numTargetShards + "]");
}
// this is just an additional assertion that ensures we are a factor of the routing num shards.
assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0;
return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor);
assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard
|| getRoutingFactor(numTargetShards, routingNumShards) >= 0;
}
/**

43
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -379,15 +379,24 @@ public class MetaDataCreateIndexService extends AbstractComponent {
indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index());
final Settings idxSettings = indexSettingsBuilder.build();
int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings);
final int routingNumShards;
if (recoverFromIndex == null) {
Settings idxSettings = indexSettingsBuilder.build();
final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null);
final IndexMetaData sourceMetaData = recoverFromIndex == null ? null :
currentState.metaData().getIndexSafe(recoverFromIndex);
if (sourceMetaData == null || sourceMetaData.getNumberOfShards() == 1) {
// in this case we either have no index to recover from or
// we have a source index with 1 shard and without an explicit split factor
// or one that is valid in that case we can split into whatever and auto-generate a new factor.
if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) {
routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings);
} else {
routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated);
}
} else {
assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false
: "index.number_of_routing_shards should be present on the target index on resize";
final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex);
: "index.number_of_routing_shards should not be present on the target index on resize";
routingNumShards = sourceMetaData.getRoutingNumShards();
}
// remove the setting it's temporary and is only relevant once we create the index
@ -408,7 +417,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
* the maximum primary term on all the shards in the source index. This ensures that we have correct
* document-level semantics regarding sequence numbers in the shrunken index.
*/
final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex);
final long primaryTerm =
IntStream
.range(0, sourceMetaData.getNumberOfShards())
@ -717,4 +725,27 @@ public class MetaDataCreateIndexService extends AbstractComponent {
.put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName())
.put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID());
}
/**
* Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will
* allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024
* the less default split operations are supported
*/
public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) {
if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) {
// only select this automatically for indices that are created on or after 7.0 this will prevent this new behaviour
// until we have a fully upgraded cluster. Additionally it will make integratin testing easier since mixed clusters
// will always have the behavior of the min node in the cluster.
//
// We use as a default number of routing shards the higher number that can be expressed
// as {@code numShards * 2^x`} that is less than or equal to the maximum number of shards: 1024.
int log2MaxNumShards = 10; // logBase2(1024)
int log2NumShards = 32 - Integer.numberOfLeadingZeros(numShards - 1); // ceil(logBase2(numShards))
int numSplits = log2MaxNumShards - log2NumShards;
numSplits = Math.max(1, numSplits); // Ensure the index can be split at least once
return numShards * 1 << numSplits;
} else {
return numShards;
}
}
}

1
core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
View File

@ -317,6 +317,7 @@ public class CreateIndexIT extends ESIntegTestCase {
response = prepareCreate("test_" + shards + "_" + partitionSize)
.setSettings(Settings.builder()
.put("index.number_of_shards", shards)
.put("index.number_of_routing_shards", shards)
.put("index.routing_partition_size", partitionSize))
.execute().actionGet();
} catch (IllegalStateException | IllegalArgumentException e) {

51
core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.cluster.routing.ShardRouting;
@ -66,7 +67,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.IntFunction;
import java.util.stream.IntStream;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@ -90,19 +90,38 @@ public class SplitIndexIT extends ESIntegTestCase {
public void testCreateSplitIndexToN() throws IOException {
int[][] possibleShardSplits = new int[][]{{2, 4, 8}, {3, 6, 12}, {1, 2, 4}};
int[] shardSplits = randomFrom(possibleShardSplits);
assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]);
assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]);
splitToN(shardSplits[0], shardSplits[1], shardSplits[2]);
}
public void testSplitFromOneToN() {
splitToN(1, 5, 10);
client().admin().indices().prepareDelete("*").get();
int randomSplit = randomIntBetween(2, 6);
splitToN(1, randomSplit, randomSplit * 2);
}
private void splitToN(int sourceShards, int firstSplitShards, int secondSplitShards) {
assertEquals(sourceShards, (sourceShards * firstSplitShards) / firstSplitShards);
assertEquals(firstSplitShards, (firstSplitShards * secondSplitShards) / secondSplitShards);
internalCluster().ensureAtLeastNumDataNodes(2);
final boolean useRouting = randomBoolean();
final boolean useNested = randomBoolean();
final boolean useMixedRouting = useRouting ? randomBoolean() : false;
CreateIndexRequestBuilder createInitialIndex = prepareCreate("source");
final int routingShards = shardSplits[2] * randomIntBetween(1, 10);
Settings.Builder settings = Settings.builder().put(indexSettings())
.put("number_of_shards", shardSplits[0])
.put("index.number_of_routing_shards", routingShards);
if (useRouting && useMixedRouting == false && randomBoolean()) {
settings.put("index.routing_partition_size", randomIntBetween(1, routingShards - 1));
Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", sourceShards);
final boolean useRoutingPartition;
if (randomBoolean()) {
// randomly set the value manually
int routingShards = secondSplitShards * randomIntBetween(1, 10);
settings.put("index.number_of_routing_shards", routingShards);
useRoutingPartition = false;
} else {
useRoutingPartition = randomBoolean();
}
if (useRouting && useMixedRouting == false && useRoutingPartition) {
settings.put("index.routing_partition_size",
randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards(sourceShards, Version.CURRENT)-1));
if (useNested) {
createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested");
} else {
@ -172,11 +191,15 @@ public class SplitIndexIT extends ESIntegTestCase {
.setSettings(Settings.builder()
.put("index.blocks.write", true)).get();
ensureGreen();
Settings.Builder firstSplitSettingsBuilder = Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", firstSplitShards);
if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard
firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards);
}
assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split")
.setResizeType(ResizeType.SPLIT)
.setSettings(Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", shardSplits[1]).build()).get());
.setSettings(firstSplitSettingsBuilder.build()).get());
ensureGreen();
assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
@ -204,7 +227,7 @@ public class SplitIndexIT extends ESIntegTestCase {
.setResizeType(ResizeType.SPLIT)
.setSettings(Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", shardSplits[2]).build()).get());
.put("index.number_of_shards", secondSplitShards).build()).get());
ensureGreen();
assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
// let it be allocated anywhere and bump replicas
@ -340,7 +363,6 @@ public class SplitIndexIT extends ESIntegTestCase {
prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
.put("number_of_shards", 1)
.put("index.version.created", version)
.put("index.number_of_routing_shards", 2)
).get();
final int docs = randomIntBetween(0, 128);
for (int i = 0; i < docs; i++) {
@ -443,7 +465,6 @@ public class SplitIndexIT extends ESIntegTestCase {
Settings.builder()
.put(indexSettings())
.put("sort.field", "id")
.put("index.number_of_routing_shards", 16)
.put("sort.order", "desc")
.put("number_of_shards", 2)
.put("number_of_replicas", 0)

66
core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
View File

@ -51,10 +51,13 @@ import static java.util.Collections.emptyMap;
public class TransportResizeActionTests extends ESTestCase {
private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) {
return createClusterState(name, numShards, numReplicas, numShards, settings);
}
private ClusterState createClusterState(String name, int numShards, int numReplicas, int numRoutingShards, Settings settings) {
MetaData.Builder metaBuilder = MetaData.builder();
IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT)
.put(settings))
.numberOfShards(numShards).numberOfReplicas(numReplicas).build();
.numberOfShards(numShards).numberOfReplicas(numReplicas).setRoutingNumShards(numRoutingShards).build();
metaBuilder.put(indexMetaData, false);
MetaData metaData = metaBuilder.build();
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
@ -108,6 +111,67 @@ public class TransportResizeActionTests extends ESTestCase {
(i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), "source", "target");
}
public void testPassNumRoutingShards() {
ClusterState clusterState = ClusterState.builder(createClusterState("source", 1, 0,
Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
.build();
AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
// now we start the shard
routingTable = service.applyStartedShards(clusterState,
routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
ResizeRequest resizeRequest = new ResizeRequest("target", "source");
resizeRequest.setResizeType(ResizeType.SPLIT);
resizeRequest.getTargetIndexRequest()
.settings(Settings.builder().put("index.number_of_shards", 2).build());
TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target");
resizeRequest.getTargetIndexRequest()
.settings(Settings.builder()
.put("index.number_of_routing_shards", randomIntBetween(2, 10))
.put("index.number_of_shards", 2)
.build());
TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target");
}
public void testPassNumRoutingShardsAndFail() {
int numShards = randomIntBetween(2, 100);
ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, numShards * 4,
Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
.build();
AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
// now we start the shard
routingTable = service.applyStartedShards(clusterState,
routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
ResizeRequest resizeRequest = new ResizeRequest("target", "source");
resizeRequest.setResizeType(ResizeType.SPLIT);
resizeRequest.getTargetIndexRequest()
.settings(Settings.builder().put("index.number_of_shards", numShards * 2).build());
TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target");
resizeRequest.getTargetIndexRequest()
.settings(Settings.builder()
.put("index.number_of_shards", numShards * 2)
.put("index.number_of_routing_shards", numShards * 2).build());
ClusterState finalState = clusterState;
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
() -> TransportResizeAction.prepareCreateIndexRequest(resizeRequest, finalState, null, "source", "target"));
assertEquals("cannot provide index.number_of_routing_shards on resize", iae.getMessage());
}
public void testShrinkIndexSettings() {
String indexName = randomAlphaOfLength(10);
// create one that won't fail

36
core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -34,7 +34,6 @@ import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllo
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.ResourceAlreadyExistsException;
@ -299,4 +298,39 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
.getDefault(Settings.EMPTY)).build()));
assertThat(e.getMessage(), endsWith(errorMessage));
}
public void testCalculateNumRoutingShards() {
assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT));
assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT));
assertEquals(768, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT));
assertEquals(576, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT));
assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT));
assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT));
assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT));
Version latestV6 = VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1);
int numShards = randomIntBetween(1, 1000);
assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, latestV6));
assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards,
VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), latestV6)));
for (int i = 0; i < 1000; i++) {
int randomNumShards = randomIntBetween(1, 10000);
int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT);
if (numRoutingShards <= 1024) {
assertTrue("numShards: " + randomNumShards, randomNumShards < 513);
assertTrue("numRoutingShards: " + numRoutingShards, numRoutingShards > 512);
} else {
assertEquals("numShards: " + randomNumShards, numRoutingShards / 2, randomNumShards);
}
double ratio = numRoutingShards / randomNumShards;
int intRatio = (int) ratio;
assertEquals(ratio, (double)(intRatio), 0.0d);
assertTrue(1 < ratio);
assertTrue(ratio <= 1024);
assertEquals(0, intRatio % 2);
assertEquals("ratio is not a power of two", intRatio, Integer.highestOneBit(intRatio));
}
}
}

1
core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
View File

@ -52,7 +52,6 @@ import static org.hamcrest.object.HasToString.hasToString;
public class OperationRoutingTests extends ESTestCase{
public void testGenerateShardId() {
int[][] possibleValues = new int[][] {
{8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1}

6
core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
View File

@ -95,7 +95,8 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
Client client = client();
assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
.setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get());
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put("index.number_of_routing_shards", 5)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get());
indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
@ -362,7 +363,8 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
public void testCanCache() throws Exception {
Client client = client();
Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
.setSettings(settings)
.get());

6
core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
View File

@ -840,7 +840,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
// create an index with too few shards
IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class,
() -> prepareCreate("test_bad", Settings.builder()
.put("index.number_of_shards", 5))
.put("index.number_of_shards", 5)
.put("index.number_of_routing_shards", 5))
.get());
assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number "
@ -848,7 +849,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
// finally, create a valid index
prepareCreate("test_good", Settings.builder()
.put("index.number_of_shards", 7))
.put("index.number_of_shards", 7)
.put("index.number_of_routing_shards", 7))
.get();
GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get();

9
core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
View File

@ -24,6 +24,8 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ESIntegTestCase;
@ -196,6 +198,13 @@ public class AliasRoutingIT extends ESIntegTestCase {
}
@Override
public Settings indexSettings() {
Settings settings = super.indexSettings();
int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings);
return Settings.builder().put(settings).put("index.number_of_routing_shards", numShards).build();
}
public void testAliasSearchRoutingWithTwoIndices() throws Exception {
createIndex("test-a");
createIndex("test-b");

2
core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java
View File

@ -42,6 +42,7 @@ public class PartitionedRoutingIT extends ESIntegTestCase {
client().admin().indices().prepareCreate(index)
.setSettings(Settings.builder()
.put("index.number_of_shards", shards)
.put("index.number_of_routing_shards", shards)
.put("index.routing_partition_size", partitionSize))
.addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON)
.execute().actionGet();
@ -67,6 +68,7 @@ public class PartitionedRoutingIT extends ESIntegTestCase {
client().admin().indices().prepareCreate(index)
.setSettings(Settings.builder()
.put("index.number_of_shards", currentShards)
.put("index.number_of_routing_shards", currentShards)
.put("index.number_of_replicas", numberOfReplicas())
.put("index.routing_partition_size", partitionSize))
.addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON)

121
core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
View File

@ -36,8 +36,14 @@ import org.elasticsearch.action.termvectors.TermVectorsResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.OperationRouting;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESIntegTestCase;
@ -53,12 +59,27 @@ public class SimpleRoutingIT extends ESIntegTestCase {
return 2;
}
public String findNonMatchingRoutingValue(String index, String id) {
OperationRouting operationRouting = new OperationRouting(Settings.EMPTY,
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
ClusterState state = client().admin().cluster().prepareState().all().get().getState();
int routing = -1;
ShardId idShard;
ShardId routingShard;
do {
idShard = operationRouting.shardId(state, index, id, null);
routingShard = operationRouting.shardId(state, index, id, Integer.toString(++routing));
} while (idShard.getId() == routingShard.id());
return Integer.toString(routing);
}
public void testSimpleCrudRouting() throws Exception {
createIndex("test");
ensureGreen();
logger.info("--> indexing with id [1], and routing [0]");
client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
String routingValue = findNonMatchingRoutingValue("test", "1");
logger.info("--> indexing with id [1], and routing [{}]", routingValue);
client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
.get();
logger.info("--> verifying get with no routing, should not find anything");
for (int i = 0; i < 5; i++) {
@ -66,25 +87,25 @@ public class SimpleRoutingIT extends ESIntegTestCase {
}
logger.info("--> verifying get with routing, should find");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
logger.info("--> deleting with no routing, should not delete anything");
client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
logger.info("--> deleting with routing, should delete");
client().prepareDelete("test", "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
client().prepareDelete("test", "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false));
assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false));
}
logger.info("--> indexing with id [1], and routing [0]");
client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
.get();
logger.info("--> verifying get with no routing, should not find anything");
for (int i = 0; i < 5; i++) {
@ -92,16 +113,17 @@ public class SimpleRoutingIT extends ESIntegTestCase {
}
logger.info("--> verifying get with routing, should find");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
}
public void testSimpleSearchRouting() {
createIndex("test");
ensureGreen();
String routingValue = findNonMatchingRoutingValue("test", "1");
logger.info("--> indexing with id [1], and routing [0]");
client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
logger.info("--> indexing with id [1], and routing [{}]", routingValue);
client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
.get();
logger.info("--> verifying get with no routing, should not find anything");
for (int i = 0; i < 5; i++) {
@ -109,7 +131,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
}
logger.info("--> verifying get with routing, should find");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
logger.info("--> search with no routing, should fine one");
@ -125,12 +147,13 @@ public class SimpleRoutingIT extends ESIntegTestCase {
logger.info("--> search with correct routing, should find");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
}
logger.info("--> indexing with id [2], and routing [1]");
client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
String secondRoutingValue = "1";
logger.info("--> indexing with id [{}], and routing [{}]", routingValue, secondRoutingValue);
client().prepareIndex("test", "type1", routingValue).setRouting(secondRoutingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
logger.info("--> search with no routing, should fine two");
for (int i = 0; i < 5; i++) {
@ -138,28 +161,28 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
}
logger.info("--> search with 0 routing, should find one");
logger.info("--> search with {} routing, should find one", routingValue);
for (int i = 0; i < 5; i++) {
assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
}
logger.info("--> search with 1 routing, should find one");
logger.info("--> search with {} routing, should find one", secondRoutingValue);
for (int i = 0; i < 5; i++) {
assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
assertThat(client().prepareSearch().setSize(0).setRouting(secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L));
}
logger.info("--> search with 0,1 routings , should find two");
logger.info("--> search with {},{} routings , should find two", routingValue, "1");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
}
logger.info("--> search with 0,1,0 routings , should find two");
logger.info("--> search with {},{},{} routings , should find two", routingValue, secondRoutingValue, routingValue);
for (int i = 0; i < 5; i++) {
assertThat(client().prepareSearch().setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setSize(0).setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue, routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue,routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L));
}
}
@ -168,9 +191,10 @@ public class SimpleRoutingIT extends ESIntegTestCase {
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject())
.execute().actionGet();
ensureGreen();
String routingValue = findNonMatchingRoutingValue("test", "1");
logger.info("--> indexing with id [1], and routing [0]");
client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1")
logger.info("--> indexing with id [1], and routing [{}]", routingValue);
client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1")
.setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
logger.info("--> verifying get with no routing, should fail");
@ -184,7 +208,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
logger.info("--> verifying get with routing, should find");
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
logger.info("--> deleting with no routing, should fail");
@ -203,7 +227,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
}
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
}
try {
@ -213,7 +237,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class));
}
client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get();
client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get();
client().admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 5; i++) {
@ -224,12 +248,12 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
}
GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet();
GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getSourceAsMap().get("field"), equalTo("value2"));
}
client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
client().prepareDelete(indexOrAlias(), "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
for (int i = 0; i < 5; i++) {
try {
@ -239,7 +263,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
}
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false));
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false));
}
}
@ -251,7 +275,6 @@ public class SimpleRoutingIT extends ESIntegTestCase {
.endObject().endObject())
.execute().actionGet();
ensureGreen();
{
BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1")
.source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
@ -320,19 +343,21 @@ public class SimpleRoutingIT extends ESIntegTestCase {
}
public void testRequiredRoutingMappingVariousAPIs() throws Exception {
client().admin().indices().prepareCreate("test").addAlias(new Alias("alias"))
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject())
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("_routing").field("required", true).endObject().endObject().endObject())
.execute().actionGet();
ensureGreen();
logger.info("--> indexing with id [1], and routing [0]");
client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").get();
logger.info("--> indexing with id [2], and routing [0]");
client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2")
String routingValue = findNonMatchingRoutingValue("test", "1");
logger.info("--> indexing with id [1], and routing [{}]", routingValue);
client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1").get();
logger.info("--> indexing with id [2], and routing [{}]", routingValue);
client().prepareIndex(indexOrAlias(), "type1", "2").setRouting(routingValue).setSource("field", "value2")
.setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
logger.info("--> verifying get with id [1] with routing [0], should succeed");
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true));
logger.info("--> verifying get with id [1], with no routing, should fail");
try {
@ -345,7 +370,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
logger.info("--> verifying explain with id [2], with routing [0], should succeed");
ExplainResponse explainResponse = client().prepareExplain(indexOrAlias(), "type1", "2")
.setQuery(QueryBuilders.matchAllQuery())
.setRouting("0").get();
.setRouting(routingValue).get();
assertThat(explainResponse.isExists(), equalTo(true));
assertThat(explainResponse.isMatch(), equalTo(true));
@ -359,7 +384,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
}
logger.info("--> verifying term vector with id [1], with routing [0], should succeed");
TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting("0").get();
TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting(routingValue).get();
assertThat(termVectorsResponse.isExists(), equalTo(true));
assertThat(termVectorsResponse.getId(), equalTo("1"));
@ -370,7 +395,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
}
UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0")
UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue)
.setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get();
assertThat(updateResponse.getId(), equalTo("1"));
assertThat(updateResponse.getVersion(), equalTo(2L));
@ -405,8 +430,8 @@ public class SimpleRoutingIT extends ESIntegTestCase {
assertThat(multiGetResponse.getResponses()[1].getFailure().getMessage(), equalTo("routing is required for [test]/[type1]/[2]"));
MultiTermVectorsResponse multiTermVectorsResponse = client().prepareMultiTermVectors()
.add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing("0"))
.add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing("0")).get();
.add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing(routingValue))
.add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing(routingValue)).get();
assertThat(multiTermVectorsResponse.getResponses().length, equalTo(2));
assertThat(multiTermVectorsResponse.getResponses()[0].getId(), equalTo("1"));
assertThat(multiTermVectorsResponse.getResponses()[0].isFailed(), equalTo(false));

2
core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
View File

@ -50,7 +50,7 @@ public class SharedSignificantTermsTestMethods {
public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException {
String type = ESTestCase.randomBoolean() ? "text" : "keyword";
String settings = "{\"index.number_of_shards\": 7, \"index.number_of_replicas\": 0}";
String settings = "{\"index.number_of_shards\": 7, \"index.number_of_routing_shards\": 7, \"index.number_of_replicas\": 0}";
index01Docs(type, settings, testCase);
testCase.ensureGreen();
testCase.logClusterState();

2
core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
View File

@ -258,7 +258,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase {
public void testExplainWithRewriteValidateQueryAllShards() throws Exception {
client().admin().indices().prepareCreate("test")
.addMapping("type1", "field", "type=text,analyzer=whitespace")
.setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2)).get();
.setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get();
// We are relying on specific routing behaviors for the result to be right, so
// we cannot randomize the number of shards or change ids here.
client().prepareIndex("test", "type1", "1")

7
docs/build.gradle
View File

@ -194,6 +194,13 @@ buildRestTests.setups['sales'] = '''
// Dummy bank account data used by getting-started.asciidoc
buildRestTests.setups['bank'] = '''
- do:
indices.create:
index: bank
body:
settings:
number_of_shards: 5
number_of_routing_shards: 5
- do:
bulk:
index: bank

4
docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc
View File

@ -60,8 +60,8 @@ The response for the above aggregation:
"aggregations": {
"centroid": {
"location": {
"lat": 51.00982963107526,
"lon": 3.9662130922079086
"lat": 51.00982963806018,
"lon": 3.9662131061777472
},
"count": 6
}

4
docs/reference/cat/segments.asciidoc
View File

@ -17,8 +17,8 @@ might look like:
["source","txt",subs="attributes,callouts"]
--------------------------------------------------
index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound
test 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
test1 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
test 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
test1 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
--------------------------------------------------
// TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat]

61
docs/reference/indices/split-index.asciidoc
View File

@ -1,23 +1,36 @@
[[indices-split-index]]
== Split Index
number_of_routing_shards
The split index API allows you to split an existing index into a new index,
where each original primary shard is split into two or more primary shards in
the new index.
The number of times the index can be split (and the number of shards that each
original shard can be split into) is determined by the
`index.number_of_routing_shards` setting. The number of routing shards
specifies the hashing space that is used internally to distribute documents
across shards with consistent hashing. For instance, a 5 shard index with
`number_of_routing_shards` set to `30` (`5 x 2 x 3`) could be split by a
factor of `2` or `3`. In other words, it could be split as follows:
* `5` -> `10` -> `30` (split by 2, then by 3)
* `5` -> `15` -> `30` (split by 3, then by 2)
* `5` -> `30` (split by 6)
While you can set the `index.number_of_routing_shards` setting explicitly at
index creation time, the default value depends upon the number of primary
shards in the original index. The default is designed to allow you to split
by factors of 2 up to a maximum of 1024 shards. However, the original number
of primary shards must taken into account. For instance, an index created
with 5 primary shards could be split into 10, 20, 40, 80, 160, 320, or a
maximum of 740 shards (with a single split action or multiple split actions).
If the original index contains one primary shard (or a multi-shard index has
been <<indices-shrink-index,shrunk>> down to a single primary shard), then the
index may by split into an arbitrary number of shards greater than 1. The
properties of the default number of routing shards will then apply to the
newly split index.
The split index API allows you to split an existing index into a new index
with multiple of it's primary shards. Similarly to the <<indices-shrink-index,Shrink API>>
where the number of primary shards in the shrunk index must be a factor of the source index.
The `_split` API requires the source index to be created with a specific number of routing shards
in order to be split in the future. (Note: this requirement might be remove in future releases)
The number of routing shards specify the hashing space that is used internally to distribute documents
across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch
uses today.
For example an index with `8` primary shards and a `index.number_of_routing_shards` of `32`
can be split into `16` and `32` primary shards. An index with `1` primary shard
and `index.number_of_routing_shards` of `64` can be split into `2`, `4`, `8`, `16`, `32` or `64`.
The same works for non power of two routing shards ie. an index with `1` primary shard and
`index.number_of_routing_shards` set to `15` can be split into `3` and `15` or alternatively`5` and `15`.
The number of shards in the split index must always be a factor of `index.number_of_routing_shards`
in the source index. Before splitting, a (primary) copy of every shard in the index must be active in the cluster.
Splitting works as follows:
@ -29,7 +42,7 @@ Splitting works as follows:
into the new index, which is a much more time consuming process.)
* Once the low level files are created all documents will be `hashed` again to delete
documents that belong in a different shard.
documents that belong to a different shard.
* Finally, it recovers the target index as though it were a closed index which
had just been re-opened.
@ -37,23 +50,19 @@ Splitting works as follows:
[float]
=== Preparing an index for splitting
Create an index with a routing shards factor:
Create a new index:
[source,js]
--------------------------------------------------
PUT my_source_index
{
"settings": {
"index.number_of_shards" : 1,
"index.number_of_routing_shards" : 2 <1>
"index.number_of_shards" : 1
}
}
-------------------------------------------------
// CONSOLE
<1> Allows to split the index into two shards or in other words, it allows
for a single split operation.
In order to split an index, the index must be marked as read-only,
and have <<cluster-health,health>> `green`.
@ -75,7 +84,7 @@ PUT /my_source_index/_settings
changes like deleting the index.
[float]
=== Spitting an index
=== Splitting an index
To split `my_source_index` into a new index called `my_target_index`, issue
the following request:
@ -102,7 +111,7 @@ Indices can only be split if they satisfy the following requirements:
* the target index must not exist
* The index must have less primary shards than the target index.
* The source index must have fewer primary shards than the target index.
* The number of primary shards in the target index must be a factor of the
number of primary shards in the source index.
@ -128,7 +137,7 @@ POST my_source_index/_split/my_target_index
}
--------------------------------------------------
// CONSOLE
// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_routing_shards" : 5, "index.number_of_shards": "1"}}\n/]
// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_shards": "1"}}\n/]
<1> The number of shards in the target index. This must be a factor of the
number of shards in the source index.

8
docs/reference/migration/migrate_7_0/indices.asciidoc
View File

@ -37,3 +37,11 @@ To safeguard against creating too many tokens, the difference between `max_shing
limit can be changed with the index setting `index.max_shingle_diff`. Note that if the limit is
exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation
warning is logged.
==== Document distribution changes
Indices created with version `7.0.0` onwards will have an automatic `index.number_of_routing_shards`
value set. This might change how documents are distributed across shards depending on how many
shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the
`index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time.
Note: if the number of routing shards equals the number of shards `_split` operations are not supported.

42
docs/reference/query-dsl/percolate-query.asciidoc
View File

@ -380,27 +380,6 @@ This will yield the following response.
"total": 2,
"max_score": 0.5753642,
"hits": [
{
"_index": "my-index",
"_type": "doc",
"_id": "4",
"_score": 0.5753642,
"_source": {
"query": {
"match": {
"message": "lazy dog"
}
}
},
"highlight": {
"message": [
"The quick brown fox jumps over the <em>lazy</em> <em>dog</em>" <1>
]
},
"fields" : {
"_percolator_document_slot" : [0]
}
},
{
"_index": "my-index",
"_type": "doc",
@ -421,6 +400,27 @@ This will yield the following response.
"fields" : {
"_percolator_document_slot" : [0]
}
},
{
"_index": "my-index",
"_type": "doc",
"_id": "4",
"_score": 0.5753642,
"_source": {
"query": {
"match": {
"message": "lazy dog"
}
}
},
"highlight": {
"message": [
"The quick brown fox jumps over the <em>lazy</em> <em>dog</em>" <1>
]
},
"fields" : {
"_percolator_document_slot" : [0]
}
}
]
}

16
docs/reference/search/search-shards.asciidoc
View File

@ -100,7 +100,7 @@ And specifying the same request, this time with a routing value:
[source,js]
--------------------------------------------------
GET /twitter/_search_shards?routing=foo,baz
GET /twitter/_search_shards?routing=foo,bar
--------------------------------------------------
// CONSOLE
// TEST[s/^/PUT twitter\n/]
@ -120,9 +120,9 @@ This will yield the following result:
"index": "twitter",
"node": "JklnKbD7Tyqi9TP3_Q_tBg",
"primary": true,
"shard": 0,
"shard": 2,
"state": "STARTED",
"allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"},
"allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"},
"relocating_node": null
}
],
@ -131,9 +131,9 @@ This will yield the following result:
"index": "twitter",
"node": "JklnKbD7Tyqi9TP3_Q_tBg",
"primary": true,
"shard": 1,
"shard": 3,
"state": "STARTED",
"allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"},
"allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"},
"relocating_node": null
}
]
@ -141,9 +141,9 @@ This will yield the following result:
}
--------------------------------------------------
// TESTRESPONSE[s/"nodes": ...,/"nodes": $body.nodes,/]
// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.0.0.node/]
// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.0.0.allocation_id.id/]
// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.1.0.allocation_id.id/]
// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.1.0.node/]
// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.1.0.allocation_id.id/]
// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.0.0.allocation_id.id/]
This time the search will only be executed against two of the shards, because
routing values have been specified.

6
docs/reference/search/validate.asciidoc
View File

@ -239,19 +239,19 @@ Response:
"index": "twitter",
"shard": 2,
"valid": true,
"explanation": "(user:kimchi)^0.8333333"
"explanation": "user:kimchy~2"
},
{
"index": "twitter",
"shard": 3,
"valid": true,
"explanation": "user:kimchy"
"explanation": "(user:kimchi)^0.8333333"
},
{
"index": "twitter",
"shard": 4,
"valid": true,
"explanation": "user:kimchy~2"
"explanation": "user:kimchy"
}
]
}

1
modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml
View File

@ -7,6 +7,7 @@ setup:
body:
settings:
number_of_shards: 3
number_of_routing_shards: 3
mappings:
test:
"properties":

1
modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml
View File

@ -7,6 +7,7 @@ setup:
body:
settings:
number_of_shards: 3
number_of_routing_shards: 3
mappings:
test:
"properties":

2
modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
View File

@ -113,7 +113,7 @@
- match: { hits.total: 1 }
- length: { hits.hits: 1 }
- match: { hits.hits.0._explanation.description: "weight(otherField:foo in 0) [PerFieldSimilarity], result of:" }
- match: { hits.hits.0._explanation.description: "weight(otherField:foo in 1) [PerFieldSimilarity], result of:" }
- do:
search_template:

2
qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml
View File

@ -337,6 +337,6 @@
body:
script:
lang: painless
source: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"}
source: if (ctx._source.user == "kimchy") {ctx.op = "index"} else {ctx.op = "junk"}
- match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }

1
rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

1
rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml
View File

@ -6,6 +6,7 @@
body:
settings:
number_of_shards: 5
number_of_routing_shards: 5
mappings:
test:
_parent: { type: "foo" }

1
rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml
View File

@ -8,6 +8,7 @@
settings:
refresh_interval: -1
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:
cluster.health:

1
rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

1
rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

1
rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

1
rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

113
rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml
View File

@ -6,9 +6,9 @@ setup:
wait_for_active_shards: 1
body:
settings:
index.number_of_shards: 1
index.number_of_shards: 2
index.number_of_replicas: 0
index.number_of_routing_shards: 2
index.number_of_routing_shards: 4
- do:
index:
index: source
@ -59,7 +59,7 @@ setup:
body:
settings:
index.number_of_replicas: 0
index.number_of_shards: 2
index.number_of_shards: 4
- do:
cluster.health:
@ -100,6 +100,107 @@ setup:
- match: { _id: "3" }
- match: { _source: { foo: "hello world 3" } }
---
"Split from 1 to N":
- skip:
version: " - 6.99.99"
reason: Added in 7.0.0
- do:
indices.create:
index: source_one_shard
wait_for_active_shards: 1
body:
settings:
index.number_of_shards: 1
index.number_of_replicas: 0
- do:
index:
index: source_one_shard
type: doc
id: "1"
body: { "foo": "hello world" }
- do:
index:
index: source_one_shard
type: doc
id: "2"
body: { "foo": "hello world 2" }
- do:
index:
index: source_one_shard
type: doc
id: "3"
body: { "foo": "hello world 3" }
# make it read-only
- do:
indices.put_settings:
index: source_one_shard
body:
index.blocks.write: true
index.number_of_replicas: 0
- do:
cluster.health:
wait_for_status: green
index: source_one_shard
# now we do the actual split from 1 to 5
- do:
indices.split:
index: "source_one_shard"
target: "target"
wait_for_active_shards: 1
master_timeout: 10s
body:
settings:
index.number_of_replicas: 0
index.number_of_shards: 5
- do:
cluster.health:
wait_for_status: green
- do:
get:
index: target
type: doc
id: "1"
- match: { _index: target }
- match: { _type: doc }
- match: { _id: "1" }
- match: { _source: { foo: "hello world" } }
- do:
get:
index: target
type: doc
id: "2"
- match: { _index: target }
- match: { _type: doc }
- match: { _id: "2" }
- match: { _source: { foo: "hello world 2" } }
- do:
get:
index: target
type: doc
id: "3"
- match: { _index: target }
- match: { _type: doc }
- match: { _id: "3" }
- match: { _source: { foo: "hello world 3" } }
---
"Create illegal split indices":
- skip:
@ -117,8 +218,8 @@ setup:
body:
settings:
index.number_of_replicas: 0
index.number_of_shards: 2
index.number_of_routing_shards: 4
index.number_of_shards: 4
index.number_of_routing_shards: 8
# try to do an illegal split with illegal number_of_shards
- do:
@ -131,4 +232,4 @@ setup:
body:
settings:
index.number_of_replicas: 0
index.number_of_shards: 3
index.number_of_shards: 6

1
rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do:

4
rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml
View File

@ -3,6 +3,10 @@ setup:
- do:
indices.create:
index: test_sliced_scroll
body:
settings:
number_of_shards: 5
number_of_routing_shards: 5
- do:
index:

2
rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml
View File

@ -5,6 +5,8 @@ setup:
body:
settings:
number_of_replicas: 0
number_of_shards: 5
number_of_routing_shards: 5
mappings:
doc:
properties:

1
rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml
View File

@ -8,6 +8,7 @@
settings:
index:
number_of_shards: 5
number_of_routing_shards: 5
number_of_replicas: 0
- do: