diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 55fe1ce6942..289ebf372d8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -264,7 +264,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase { assertEquals(5, matrixStats.getFieldCount("num")); assertEquals(56d, matrixStats.getMean("num"), 0d); assertEquals(1830d, matrixStats.getVariance("num"), 0d); - assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d); + assertEquals(0.09340198804973046, matrixStats.getSkewness("num"), 0d); assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d); assertEquals(5, matrixStats.getFieldCount("num2")); assertEquals(29d, matrixStats.getMean("num2"), 0d); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 87dd9f9fa2d..c5a15be22a8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -163,9 +163,14 @@ public class TransportResizeAction extends TransportMasterNodeAction, ToXContentFragmen * @return a the source shard ID to split off from */ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + int numSourceShards = sourceIndexMetadata.getNumberOfShards(); if (shardId >= numTargetShards) { throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } - int numSourceShards = sourceIndexMetadata.getNumberOfShards(); + final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); + return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); + } + + private static void assertSplitMetadata(int numSourceShards, int numTargetShards, IndexMetaData sourceIndexMetadata) { if (numSourceShards > numTargetShards) { throw new IllegalArgumentException("the number of source shards [" + numSourceShards - + "] must be less that the number of target shards [" + numTargetShards + "]"); + + "] must be less that the number of target shards [" + numTargetShards + "]"); } - int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); // now we verify that the numRoutingShards is valid in the source index - int routingNumShards = sourceIndexMetadata.getRoutingNumShards(); + // note: if the number of shards is 1 in the source index we can just assume it's correct since from 1 we can split into anything + // this is important to special case here since we use this to validate this in various places in the code but allow to split form + // 1 to N but we never modify the sourceIndexMetadata to accommodate for that + int routingNumShards = numSourceShards == 1 ? numTargetShards : sourceIndexMetadata.getRoutingNumShards(); if (routingNumShards % numTargetShards != 0) { throw new IllegalStateException("the number of routing shards [" + routingNumShards + "] must be a multiple of the target shards [" + numTargetShards + "]"); } // this is just an additional assertion that ensures we are a factor of the routing num shards. - assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; - return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); + assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard + || getRoutingFactor(numTargetShards, routingNumShards) >= 0; } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 64f2383fe77..01783060c0b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -379,15 +379,24 @@ public class MetaDataCreateIndexService extends AbstractComponent { indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName()); indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index()); - + final Settings idxSettings = indexSettingsBuilder.build(); + int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; - if (recoverFromIndex == null) { - Settings idxSettings = indexSettingsBuilder.build(); - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); + final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); + final IndexMetaData sourceMetaData = recoverFromIndex == null ? null : + currentState.metaData().getIndexSafe(recoverFromIndex); + if (sourceMetaData == null || sourceMetaData.getNumberOfShards() == 1) { + // in this case we either have no index to recover from or + // we have a source index with 1 shard and without an explicit split factor + // or one that is valid in that case we can split into whatever and auto-generate a new factor. + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); + } else { + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + } } else { assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false - : "index.number_of_routing_shards should be present on the target index on resize"; - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); + : "index.number_of_routing_shards should not be present on the target index on resize"; routingNumShards = sourceMetaData.getRoutingNumShards(); } // remove the setting it's temporary and is only relevant once we create the index @@ -408,7 +417,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { * the maximum primary term on all the shards in the source index. This ensures that we have correct * document-level semantics regarding sequence numbers in the shrunken index. */ - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); final long primaryTerm = IntStream .range(0, sourceMetaData.getNumberOfShards()) @@ -717,4 +725,27 @@ public class MetaDataCreateIndexService extends AbstractComponent { .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } + + /** + * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will + * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 + * the less default split operations are supported + */ + public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) { + if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { + // only select this automatically for indices that are created on or after 7.0 this will prevent this new behaviour + // until we have a fully upgraded cluster. Additionally it will make integratin testing easier since mixed clusters + // will always have the behavior of the min node in the cluster. + // + // We use as a default number of routing shards the higher number that can be expressed + // as {@code numShards * 2^x`} that is less than or equal to the maximum number of shards: 1024. + int log2MaxNumShards = 10; // logBase2(1024) + int log2NumShards = 32 - Integer.numberOfLeadingZeros(numShards - 1); // ceil(logBase2(numShards)) + int numSplits = log2MaxNumShards - log2NumShards; + numSplits = Math.max(1, numSplits); // Ensure the index can be split at least once + return numShards * 1 << numSplits; + } else { + return numShards; + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 3c55d0df9c1..5385902bc36 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -317,6 +317,7 @@ public class CreateIndexIT extends ESIntegTestCase { response = prepareCreate("test_" + shards + "_" + partitionSize) .setSettings(Settings.builder() .put("index.number_of_shards", shards) + .put("index.number_of_routing_shards", shards) .put("index.routing_partition_size", partitionSize)) .execute().actionGet(); } catch (IllegalStateException | IllegalArgumentException e) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 995d86c3939..bfa6f5a2ccd 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.cluster.routing.ShardRouting; @@ -66,7 +67,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.IntFunction; import java.util.stream.IntStream; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -88,21 +88,40 @@ public class SplitIndexIT extends ESIntegTestCase { } public void testCreateSplitIndexToN() throws IOException { - int[][] possibleShardSplits = new int[][] {{2,4,8}, {3, 6, 12}, {1, 2, 4}}; + int[][] possibleShardSplits = new int[][]{{2, 4, 8}, {3, 6, 12}, {1, 2, 4}}; int[] shardSplits = randomFrom(possibleShardSplits); - assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); - assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + splitToN(shardSplits[0], shardSplits[1], shardSplits[2]); + } + + public void testSplitFromOneToN() { + splitToN(1, 5, 10); + client().admin().indices().prepareDelete("*").get(); + int randomSplit = randomIntBetween(2, 6); + splitToN(1, randomSplit, randomSplit * 2); + } + + private void splitToN(int sourceShards, int firstSplitShards, int secondSplitShards) { + + assertEquals(sourceShards, (sourceShards * firstSplitShards) / firstSplitShards); + assertEquals(firstSplitShards, (firstSplitShards * secondSplitShards) / secondSplitShards); internalCluster().ensureAtLeastNumDataNodes(2); final boolean useRouting = randomBoolean(); final boolean useNested = randomBoolean(); final boolean useMixedRouting = useRouting ? randomBoolean() : false; CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); - final int routingShards = shardSplits[2] * randomIntBetween(1, 10); - Settings.Builder settings = Settings.builder().put(indexSettings()) - .put("number_of_shards", shardSplits[0]) - .put("index.number_of_routing_shards", routingShards); - if (useRouting && useMixedRouting == false && randomBoolean()) { - settings.put("index.routing_partition_size", randomIntBetween(1, routingShards - 1)); + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", sourceShards); + final boolean useRoutingPartition; + if (randomBoolean()) { + // randomly set the value manually + int routingShards = secondSplitShards * randomIntBetween(1, 10); + settings.put("index.number_of_routing_shards", routingShards); + useRoutingPartition = false; + } else { + useRoutingPartition = randomBoolean(); + } + if (useRouting && useMixedRouting == false && useRoutingPartition) { + settings.put("index.routing_partition_size", + randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards(sourceShards, Version.CURRENT)-1)); if (useNested) { createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested"); } else { @@ -172,11 +191,15 @@ public class SplitIndexIT extends ESIntegTestCase { .setSettings(Settings.builder() .put("index.blocks.write", true)).get(); ensureGreen(); + Settings.Builder firstSplitSettingsBuilder = Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", firstSplitShards); + if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard + firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards); + } assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split") .setResizeType(ResizeType.SPLIT) - .setSettings(Settings.builder() - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", shardSplits[1]).build()).get()); + .setSettings(firstSplitSettingsBuilder.build()).get()); ensureGreen(); assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); @@ -204,7 +227,7 @@ public class SplitIndexIT extends ESIntegTestCase { .setResizeType(ResizeType.SPLIT) .setSettings(Settings.builder() .put("index.number_of_replicas", 0) - .put("index.number_of_shards", shardSplits[2]).build()).get()); + .put("index.number_of_shards", secondSplitShards).build()).get()); ensureGreen(); assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); // let it be allocated anywhere and bump replicas @@ -340,7 +363,6 @@ public class SplitIndexIT extends ESIntegTestCase { prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) .put("number_of_shards", 1) .put("index.version.created", version) - .put("index.number_of_routing_shards", 2) ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { @@ -443,7 +465,6 @@ public class SplitIndexIT extends ESIntegTestCase { Settings.builder() .put(indexSettings()) .put("sort.field", "id") - .put("index.number_of_routing_shards", 16) .put("sort.order", "desc") .put("number_of_shards", 2) .put("number_of_replicas", 0) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index b03b043f03e..bd43182f007 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -51,10 +51,13 @@ import static java.util.Collections.emptyMap; public class TransportResizeActionTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { + return createClusterState(name, numShards, numReplicas, numShards, settings); + } + private ClusterState createClusterState(String name, int numShards, int numReplicas, int numRoutingShards, Settings settings) { MetaData.Builder metaBuilder = MetaData.builder(); IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT) .put(settings)) - .numberOfShards(numShards).numberOfReplicas(numReplicas).build(); + .numberOfShards(numShards).numberOfReplicas(numReplicas).setRoutingNumShards(numRoutingShards).build(); metaBuilder.put(indexMetaData, false); MetaData metaData = metaBuilder.build(); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); @@ -108,6 +111,67 @@ public class TransportResizeActionTests extends ESTestCase { (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), "source", "target"); } + public void testPassNumRoutingShards() { + ClusterState clusterState = ClusterState.builder(createClusterState("source", 1, 0, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + resizeRequest.setResizeType(ResizeType.SPLIT); + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder().put("index.number_of_shards", 2).build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder() + .put("index.number_of_routing_shards", randomIntBetween(2, 10)) + .put("index.number_of_shards", 2) + .build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + } + + public void testPassNumRoutingShardsAndFail() { + int numShards = randomIntBetween(2, 100); + ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, numShards * 4, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + resizeRequest.setResizeType(ResizeType.SPLIT); + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder().put("index.number_of_shards", numShards * 2).build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder() + .put("index.number_of_shards", numShards * 2) + .put("index.number_of_routing_shards", numShards * 2).build()); + ClusterState finalState = clusterState; + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, + () -> TransportResizeAction.prepareCreateIndexRequest(resizeRequest, finalState, null, "source", "target")); + assertEquals("cannot provide index.number_of_routing_shards on resize", iae.getMessage()); + } + public void testShrinkIndexSettings() { String indexName = randomAlphaOfLength(10); // create one that won't fail diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index bb90ee9fee9..6074102cde3 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllo import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.ResourceAlreadyExistsException; @@ -299,4 +298,39 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { .getDefault(Settings.EMPTY)).build())); assertThat(e.getMessage(), endsWith(errorMessage)); } + + public void testCalculateNumRoutingShards() { + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); + assertEquals(768, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT)); + assertEquals(576, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT)); + assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT)); + assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT)); + + Version latestV6 = VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1); + int numShards = randomIntBetween(1, 1000); + assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, latestV6)); + assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, + VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), latestV6))); + + for (int i = 0; i < 1000; i++) { + int randomNumShards = randomIntBetween(1, 10000); + int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT); + if (numRoutingShards <= 1024) { + assertTrue("numShards: " + randomNumShards, randomNumShards < 513); + assertTrue("numRoutingShards: " + numRoutingShards, numRoutingShards > 512); + } else { + assertEquals("numShards: " + randomNumShards, numRoutingShards / 2, randomNumShards); + } + + double ratio = numRoutingShards / randomNumShards; + int intRatio = (int) ratio; + assertEquals(ratio, (double)(intRatio), 0.0d); + assertTrue(1 < ratio); + assertTrue(ratio <= 1024); + assertEquals(0, intRatio % 2); + assertEquals("ratio is not a power of two", intRatio, Integer.highestOneBit(intRatio)); + } + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index 1f8de1ca02f..dc3c4c8bc8e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -52,7 +52,6 @@ import static org.hamcrest.object.HasToString.hasToString; public class OperationRoutingTests extends ESTestCase{ - public void testGenerateShardId() { int[][] possibleValues = new int[][] { {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index 1884361a47f..70a633f02f4 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -95,7 +95,8 @@ public class IndicesRequestCacheIT extends ESIntegTestCase { Client client = client(); assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put("index.number_of_routing_shards", 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), @@ -362,7 +363,8 @@ public class IndicesRequestCacheIT extends ESIntegTestCase { public void testCanCache() throws Exception { Client client = client(); Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(settings) .get()); diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index e81538b9057..9a09ba215fb 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -840,7 +840,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { // create an index with too few shards IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class, () -> prepareCreate("test_bad", Settings.builder() - .put("index.number_of_shards", 5)) + .put("index.number_of_shards", 5) + .put("index.number_of_routing_shards", 5)) .get()); assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number " @@ -848,7 +849,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { // finally, create a valid index prepareCreate("test_good", Settings.builder() - .put("index.number_of_shards", 7)) + .put("index.number_of_shards", 7) + .put("index.number_of_routing_shards", 7)) .get(); GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get(); diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index b99353812c8..14ec800c3a6 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -196,6 +198,13 @@ public class AliasRoutingIT extends ESIntegTestCase { } + @Override + public Settings indexSettings() { + Settings settings = super.indexSettings(); + int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings); + return Settings.builder().put(settings).put("index.number_of_routing_shards", numShards).build(); + } + public void testAliasSearchRoutingWithTwoIndices() throws Exception { createIndex("test-a"); createIndex("test-b"); diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 07a73a09f4a..74fbcc20a02 100644 --- a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -42,6 +42,7 @@ public class PartitionedRoutingIT extends ESIntegTestCase { client().admin().indices().prepareCreate(index) .setSettings(Settings.builder() .put("index.number_of_shards", shards) + .put("index.number_of_routing_shards", shards) .put("index.routing_partition_size", partitionSize)) .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON) .execute().actionGet(); @@ -67,6 +68,7 @@ public class PartitionedRoutingIT extends ESIntegTestCase { client().admin().indices().prepareCreate(index) .setSettings(Settings.builder() .put("index.number_of_shards", currentShards) + .put("index.number_of_routing_shards", currentShards) .put("index.number_of_replicas", numberOfReplicas()) .put("index.routing_partition_size", partitionSize)) .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON) diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 031bac145a3..84caed948a2 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -36,8 +36,14 @@ import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -53,12 +59,27 @@ public class SimpleRoutingIT extends ESIntegTestCase { return 2; } + public String findNonMatchingRoutingValue(String index, String id) { + OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + ClusterState state = client().admin().cluster().prepareState().all().get().getState(); + int routing = -1; + ShardId idShard; + ShardId routingShard; + do { + idShard = operationRouting.shardId(state, index, id, null); + routingShard = operationRouting.shardId(state, index, id, Integer.toString(++routing)); + } while (idShard.getId() == routingShard.id()); + + return Integer.toString(routing); + } + public void testSimpleCrudRouting() throws Exception { createIndex("test"); ensureGreen(); - - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + String routingValue = findNonMatchingRoutingValue("test", "1"); + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -66,25 +87,25 @@ public class SimpleRoutingIT extends ESIntegTestCase { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with no routing, should not delete anything"); client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with routing, should delete"); - client().prepareDelete("test", "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + client().prepareDelete("test", "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false)); } logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -92,16 +113,17 @@ public class SimpleRoutingIT extends ESIntegTestCase { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } } public void testSimpleSearchRouting() { createIndex("test"); ensureGreen(); + String routingValue = findNonMatchingRoutingValue("test", "1"); - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -109,7 +131,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> search with no routing, should fine one"); @@ -125,12 +147,13 @@ public class SimpleRoutingIT extends ESIntegTestCase { logger.info("--> search with correct routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> indexing with id [2], and routing [1]"); - client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + String secondRoutingValue = "1"; + logger.info("--> indexing with id [{}], and routing [{}]", routingValue, secondRoutingValue); + client().prepareIndex("test", "type1", routingValue).setRouting(secondRoutingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -138,28 +161,28 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } - logger.info("--> search with 0 routing, should find one"); + logger.info("--> search with {} routing, should find one", routingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with 1 routing, should find one"); + logger.info("--> search with {} routing, should find one", secondRoutingValue); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with 0,1 routings , should find two"); + logger.info("--> search with {},{} routings , should find two", routingValue, "1"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } - logger.info("--> search with 0,1,0 routings , should find two"); + logger.info("--> search with {},{},{} routings , should find two", routingValue, secondRoutingValue, routingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue, routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue,routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } } @@ -168,9 +191,10 @@ public class SimpleRoutingIT extends ESIntegTestCase { .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) .execute().actionGet(); ensureGreen(); + String routingValue = findNonMatchingRoutingValue("test", "1"); - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1") + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should fail"); @@ -184,7 +208,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with no routing, should fail"); @@ -203,7 +227,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } try { @@ -213,7 +237,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); } - client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); client().admin().indices().prepareRefresh().execute().actionGet(); for (int i = 0; i < 5; i++) { @@ -224,12 +248,12 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet(); + GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getSourceAsMap().get("field"), equalTo("value2")); } - client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + client().prepareDelete(indexOrAlias(), "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { try { @@ -239,7 +263,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false)); } } @@ -251,7 +275,6 @@ public class SimpleRoutingIT extends ESIntegTestCase { .endObject().endObject()) .execute().actionGet(); ensureGreen(); - { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1") .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); @@ -320,19 +343,21 @@ public class SimpleRoutingIT extends ESIntegTestCase { } public void testRequiredRoutingMappingVariousAPIs() throws Exception { + client().admin().indices().prepareCreate("test").addAlias(new Alias("alias")) - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) + .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("_routing").field("required", true).endObject().endObject().endObject()) .execute().actionGet(); ensureGreen(); - - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").get(); - logger.info("--> indexing with id [2], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2") + String routingValue = findNonMatchingRoutingValue("test", "1"); + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1").get(); + logger.info("--> indexing with id [2], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "2").setRouting(routingValue).setSource("field", "value2") .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with id [1] with routing [0], should succeed"); - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); logger.info("--> verifying get with id [1], with no routing, should fail"); try { @@ -345,7 +370,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { logger.info("--> verifying explain with id [2], with routing [0], should succeed"); ExplainResponse explainResponse = client().prepareExplain(indexOrAlias(), "type1", "2") .setQuery(QueryBuilders.matchAllQuery()) - .setRouting("0").get(); + .setRouting(routingValue).get(); assertThat(explainResponse.isExists(), equalTo(true)); assertThat(explainResponse.isMatch(), equalTo(true)); @@ -359,7 +384,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { } logger.info("--> verifying term vector with id [1], with routing [0], should succeed"); - TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting("0").get(); + TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting(routingValue).get(); assertThat(termVectorsResponse.isExists(), equalTo(true)); assertThat(termVectorsResponse.getId(), equalTo("1")); @@ -370,7 +395,7 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0") + UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue) .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.getVersion(), equalTo(2L)); @@ -405,8 +430,8 @@ public class SimpleRoutingIT extends ESIntegTestCase { assertThat(multiGetResponse.getResponses()[1].getFailure().getMessage(), equalTo("routing is required for [test]/[type1]/[2]")); MultiTermVectorsResponse multiTermVectorsResponse = client().prepareMultiTermVectors() - .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing("0")) - .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing("0")).get(); + .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing(routingValue)) + .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing(routingValue)).get(); assertThat(multiTermVectorsResponse.getResponses().length, equalTo(2)); assertThat(multiTermVectorsResponse.getResponses()[0].getId(), equalTo("1")); assertThat(multiTermVectorsResponse.getResponses()[0].isFailed(), equalTo(false)); diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index e5081481859..6db737793fe 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -50,7 +50,7 @@ public class SharedSignificantTermsTestMethods { public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException { String type = ESTestCase.randomBoolean() ? "text" : "keyword"; - String settings = "{\"index.number_of_shards\": 7, \"index.number_of_replicas\": 0}"; + String settings = "{\"index.number_of_shards\": 7, \"index.number_of_routing_shards\": 7, \"index.number_of_replicas\": 0}"; index01Docs(type, settings, testCase); testCase.ensureGreen(); testCase.logClusterState(); diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index a87f428fec5..3c4666148d8 100644 --- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -258,7 +258,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { public void testExplainWithRewriteValidateQueryAllShards() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type1", "field", "type=text,analyzer=whitespace") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2)).get(); + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get(); // We are relying on specific routing behaviors for the result to be right, so // we cannot randomize the number of shards or change ids here. client().prepareIndex("test", "type1", "1") diff --git a/docs/build.gradle b/docs/build.gradle index 6084561833a..65d717958ea 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -194,6 +194,13 @@ buildRestTests.setups['sales'] = ''' // Dummy bank account data used by getting-started.asciidoc buildRestTests.setups['bank'] = ''' + - do: + indices.create: + index: bank + body: + settings: + number_of_shards: 5 + number_of_routing_shards: 5 - do: bulk: index: bank diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc index 7a355c7b126..59cadf1518e 100644 --- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -60,8 +60,8 @@ The response for the above aggregation: "aggregations": { "centroid": { "location": { - "lat": 51.00982963107526, - "lon": 3.9662130922079086 + "lat": 51.00982963806018, + "lon": 3.9662131061777472 }, "count": 6 } diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 05d86758a8c..88fb18b3637 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -17,8 +17,8 @@ might look like: ["source","txt",subs="attributes,callouts"] -------------------------------------------------- index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound -test 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -test1 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test1 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -------------------------------------------------- // TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 467c09baa24..4a7e81e4f4d 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -1,23 +1,36 @@ [[indices-split-index]] == Split Index -number_of_routing_shards +The split index API allows you to split an existing index into a new index, +where each original primary shard is split into two or more primary shards in +the new index. + +The number of times the index can be split (and the number of shards that each +original shard can be split into) is determined by the +`index.number_of_routing_shards` setting. The number of routing shards +specifies the hashing space that is used internally to distribute documents +across shards with consistent hashing. For instance, a 5 shard index with +`number_of_routing_shards` set to `30` (`5 x 2 x 3`) could be split by a +factor of `2` or `3`. In other words, it could be split as follows: + +* `5` -> `10` -> `30` (split by 2, then by 3) +* `5` -> `15` -> `30` (split by 3, then by 2) +* `5` -> `30` (split by 6) + +While you can set the `index.number_of_routing_shards` setting explicitly at +index creation time, the default value depends upon the number of primary +shards in the original index. The default is designed to allow you to split +by factors of 2 up to a maximum of 1024 shards. However, the original number +of primary shards must taken into account. For instance, an index created +with 5 primary shards could be split into 10, 20, 40, 80, 160, 320, or a +maximum of 740 shards (with a single split action or multiple split actions). + +If the original index contains one primary shard (or a multi-shard index has +been <> down to a single primary shard), then the +index may by split into an arbitrary number of shards greater than 1. The +properties of the default number of routing shards will then apply to the +newly split index. -The split index API allows you to split an existing index into a new index -with multiple of it's primary shards. Similarly to the <> -where the number of primary shards in the shrunk index must be a factor of the source index. -The `_split` API requires the source index to be created with a specific number of routing shards -in order to be split in the future. (Note: this requirement might be remove in future releases) -The number of routing shards specify the hashing space that is used internally to distribute documents -across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch -uses today. -For example an index with `8` primary shards and a `index.number_of_routing_shards` of `32` -can be split into `16` and `32` primary shards. An index with `1` primary shard -and `index.number_of_routing_shards` of `64` can be split into `2`, `4`, `8`, `16`, `32` or `64`. -The same works for non power of two routing shards ie. an index with `1` primary shard and -`index.number_of_routing_shards` set to `15` can be split into `3` and `15` or alternatively`5` and `15`. -The number of shards in the split index must always be a factor of `index.number_of_routing_shards` -in the source index. Before splitting, a (primary) copy of every shard in the index must be active in the cluster. Splitting works as follows: @@ -29,7 +42,7 @@ Splitting works as follows: into the new index, which is a much more time consuming process.) * Once the low level files are created all documents will be `hashed` again to delete - documents that belong in a different shard. + documents that belong to a different shard. * Finally, it recovers the target index as though it were a closed index which had just been re-opened. @@ -37,23 +50,19 @@ Splitting works as follows: [float] === Preparing an index for splitting -Create an index with a routing shards factor: +Create a new index: [source,js] -------------------------------------------------- PUT my_source_index { - "settings": { - "index.number_of_shards" : 1, - "index.number_of_routing_shards" : 2 <1> - } + "settings": { + "index.number_of_shards" : 1 + } } ------------------------------------------------- // CONSOLE -<1> Allows to split the index into two shards or in other words, it allows - for a single split operation. - In order to split an index, the index must be marked as read-only, and have <> `green`. @@ -75,7 +84,7 @@ PUT /my_source_index/_settings changes like deleting the index. [float] -=== Spitting an index +=== Splitting an index To split `my_source_index` into a new index called `my_target_index`, issue the following request: @@ -102,7 +111,7 @@ Indices can only be split if they satisfy the following requirements: * the target index must not exist -* The index must have less primary shards than the target index. +* The source index must have fewer primary shards than the target index. * The number of primary shards in the target index must be a factor of the number of primary shards in the source index. @@ -128,7 +137,7 @@ POST my_source_index/_split/my_target_index } -------------------------------------------------- // CONSOLE -// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_routing_shards" : 5, "index.number_of_shards": "1"}}\n/] +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_shards": "1"}}\n/] <1> The number of shards in the target index. This must be a factor of the number of shards in the source index. @@ -162,4 +171,4 @@ replicas and may decide to relocate the primary shard to another node. Because the split operation creates a new index to split the shards to, the <> setting -on index creation applies to the split index action as well. +on index creation applies to the split index action as well. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc index de16498c84a..92f56a2ddbb 100644 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ b/docs/reference/migration/migrate_7_0/indices.asciidoc @@ -36,4 +36,12 @@ To safeguard against creating too many tokens, the difference between `max_shing `min_shingle_size` in `ShingleTokenFilter` has been limited to 3. This default limit can be changed with the index setting `index.max_shingle_diff`. Note that if the limit is exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. \ No newline at end of file +warning is logged. + +==== Document distribution changes + +Indices created with version `7.0.0` onwards will have an automatic `index.number_of_routing_shards` +value set. This might change how documents are distributed across shards depending on how many +shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the +`index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time. +Note: if the number of routing shards equals the number of shards `_split` operations are not supported. \ No newline at end of file diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index f5d77934075..a2c91d697fd 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -380,27 +380,6 @@ This will yield the following response. "total": 2, "max_score": 0.5753642, "hits": [ - { - "_index": "my-index", - "_type": "doc", - "_id": "4", - "_score": 0.5753642, - "_source": { - "query": { - "match": { - "message": "lazy dog" - } - } - }, - "highlight": { - "message": [ - "The quick brown fox jumps over the lazy dog" <1> - ] - }, - "fields" : { - "_percolator_document_slot" : [0] - } - }, { "_index": "my-index", "_type": "doc", @@ -421,6 +400,27 @@ This will yield the following response. "fields" : { "_percolator_document_slot" : [0] } + }, + { + "_index": "my-index", + "_type": "doc", + "_id": "4", + "_score": 0.5753642, + "_source": { + "query": { + "match": { + "message": "lazy dog" + } + } + }, + "highlight": { + "message": [ + "The quick brown fox jumps over the lazy dog" <1> + ] + }, + "fields" : { + "_percolator_document_slot" : [0] + } } ] } diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index b20117bb75d..1a7c4554576 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -100,7 +100,7 @@ And specifying the same request, this time with a routing value: [source,js] -------------------------------------------------- -GET /twitter/_search_shards?routing=foo,baz +GET /twitter/_search_shards?routing=foo,bar -------------------------------------------------- // CONSOLE // TEST[s/^/PUT twitter\n/] @@ -120,9 +120,9 @@ This will yield the following result: "index": "twitter", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, - "shard": 0, + "shard": 2, "state": "STARTED", - "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"}, + "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"}, "relocating_node": null } ], @@ -131,9 +131,9 @@ This will yield the following result: "index": "twitter", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, - "shard": 1, + "shard": 3, "state": "STARTED", - "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"}, + "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"}, "relocating_node": null } ] @@ -141,9 +141,9 @@ This will yield the following result: } -------------------------------------------------- // TESTRESPONSE[s/"nodes": ...,/"nodes": $body.nodes,/] -// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.0.0.node/] -// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.0.0.allocation_id.id/] -// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.1.0.allocation_id.id/] +// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.1.0.node/] +// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.1.0.allocation_id.id/] +// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.0.0.allocation_id.id/] This time the search will only be executed against two of the shards, because routing values have been specified. diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 7218f6e6b23..a9de086cbbb 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -239,19 +239,19 @@ Response: "index": "twitter", "shard": 2, "valid": true, - "explanation": "(user:kimchi)^0.8333333" + "explanation": "user:kimchy~2" }, { "index": "twitter", "shard": 3, "valid": true, - "explanation": "user:kimchy" + "explanation": "(user:kimchi)^0.8333333" }, { "index": "twitter", "shard": 4, "valid": true, - "explanation": "user:kimchy~2" + "explanation": "user:kimchy" } ] } diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml index 782e5d5ebbc..fbc73974f8a 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml @@ -7,6 +7,7 @@ setup: body: settings: number_of_shards: 3 + number_of_routing_shards: 3 mappings: test: "properties": diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml index b4abf41a515..dff452d43cf 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml @@ -7,6 +7,7 @@ setup: body: settings: number_of_shards: 3 + number_of_routing_shards: 3 mappings: test: "properties": diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index d2608a48e73..53b52cb3b8b 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -113,7 +113,7 @@ - match: { hits.total: 1 } - length: { hits.hits: 1 } - - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 0) [PerFieldSimilarity], result of:" } + - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 1) [PerFieldSimilarity], result of:" } - do: search_template: diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml index ea9fa33e6a9..aee2b2fb057 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml +++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml @@ -337,6 +337,6 @@ body: script: lang: painless - source: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"} + source: if (ctx._source.user == "kimchy") {ctx.op = "index"} else {ctx.op = "junk"} - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml index 44447ebee8c..752489f722c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml index d9aa6870460..82fc8a325d6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml @@ -6,6 +6,7 @@ body: settings: number_of_shards: 5 + number_of_routing_shards: 5 mappings: test: _parent: { type: "foo" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml index 589d2cf77a3..f86c7250a37 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml @@ -8,6 +8,7 @@ settings: refresh_interval: -1 number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: cluster.health: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml index 6b55a3bee37..25315628d7e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml index 94a40c0437a..276346cda4f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml index bd9a2918447..b4eebfa8ab9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml index 7b3c21df4e0..5b0cf94f423 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 7f0b294a69d..ee40cc36347 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -6,9 +6,9 @@ setup: wait_for_active_shards: 1 body: settings: - index.number_of_shards: 1 + index.number_of_shards: 2 index.number_of_replicas: 0 - index.number_of_routing_shards: 2 + index.number_of_routing_shards: 4 - do: index: index: source @@ -59,7 +59,7 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 2 + index.number_of_shards: 4 - do: cluster.health: @@ -100,6 +100,107 @@ setup: - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } + +--- +"Split from 1 to N": + - skip: + version: " - 6.99.99" + reason: Added in 7.0.0 + - do: + indices.create: + index: source_one_shard + wait_for_active_shards: 1 + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + - do: + index: + index: source_one_shard + type: doc + id: "1" + body: { "foo": "hello world" } + + - do: + index: + index: source_one_shard + type: doc + id: "2" + body: { "foo": "hello world 2" } + + - do: + index: + index: source_one_shard + type: doc + id: "3" + body: { "foo": "hello world 3" } + + # make it read-only + - do: + indices.put_settings: + index: source_one_shard + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source_one_shard + + # now we do the actual split from 1 to 5 + - do: + indices.split: + index: "source_one_shard" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 5 + + - do: + cluster.health: + wait_for_status: green + + - do: + get: + index: target + type: doc + id: "1" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "1" } + - match: { _source: { foo: "hello world" } } + + + - do: + get: + index: target + type: doc + id: "2" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "2" } + - match: { _source: { foo: "hello world 2" } } + + + - do: + get: + index: target + type: doc + id: "3" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "3" } + - match: { _source: { foo: "hello world 3" } } + + + --- "Create illegal split indices": - skip: @@ -117,8 +218,8 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 2 - index.number_of_routing_shards: 4 + index.number_of_shards: 4 + index.number_of_routing_shards: 8 # try to do an illegal split with illegal number_of_shards - do: @@ -131,4 +232,4 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 3 + index.number_of_shards: 6 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml index 7196412ebf3..d550dd26657 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml index 4acc4d13232..47bcbdb83c4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml @@ -3,6 +3,10 @@ setup: - do: indices.create: index: test_sliced_scroll + body: + settings: + number_of_shards: 5 + number_of_routing_shards: 5 - do: index: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml index 785e3be8d00..a8b7f1c6299 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml @@ -5,6 +5,8 @@ setup: body: settings: number_of_replicas: 0 + number_of_shards: 5 + number_of_routing_shards: 5 mappings: doc: properties: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml index 097f49007e5..977db506710 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: