mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-25 01:19:02 +00:00
Routing: Restore shard routing.
Routing has been inadvertly changed in #5562 resulting in documents going to different shards in 1.2. This is a terrible bug because an indexing request would not necessarily go to the same shard anymore, potentially leading to duplicates. Close #6391
This commit is contained in:
Adrien Grand
parent
93552fe8f4
commit
7ab99de483
1
pom.xml
1
pom.xml
@ -1097,6 +1097,7 @@
|
||||
<!-- start excludes for Math.abs -->
|
||||
<exclude>org/elasticsearch/common/util/MathUtils.class</exclude>
|
||||
<exclude>org/elasticsearch/common/math/UnboxedMathUtils.class</exclude>
|
||||
<exclude>org/elasticsearch/cluster/routing/operation/plain/PlainOperationRouting.class</exclude>
|
||||
<!-- end excludes for Math.abs -->
|
||||
<!-- start exclude for FilteredQuery -->
|
||||
<exclude>org/elasticsearch/common/lucene/search/XFilteredQuery.class</exclude>
|
||||
|
||||
@ -35,7 +35,6 @@ import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.component.AbstractComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.math.MathUtils;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexShardMissingException;
|
||||
@ -266,12 +265,12 @@ public class PlainOperationRouting extends AbstractComponent implements Operatio
|
||||
private int shardId(ClusterState clusterState, String index, String type, @Nullable String id, @Nullable String routing) {
|
||||
if (routing == null) {
|
||||
if (!useType) {
|
||||
return MathUtils.mod(hash(id), indexMetaData(clusterState, index).numberOfShards());
|
||||
return Math.abs(hash(id) % indexMetaData(clusterState, index).numberOfShards());
|
||||
} else {
|
||||
return MathUtils.mod(hash(type, id), indexMetaData(clusterState, index).numberOfShards());
|
||||
return Math.abs(hash(type, id) % indexMetaData(clusterState, index).numberOfShards());
|
||||
}
|
||||
}
|
||||
return MathUtils.mod(hash(routing), indexMetaData(clusterState, index).numberOfShards());
|
||||
return Math.abs(hash(routing) % indexMetaData(clusterState, index).numberOfShards());
|
||||
}
|
||||
|
||||
protected int hash(String routing) {
|
||||
|
||||
@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.cluster.routing;
|
||||
|
||||
import org.elasticsearch.cluster.ClusterName;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.cluster.routing.operation.OperationRouting;
|
||||
import org.elasticsearch.node.internal.InternalNode;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
public class RoutingBackwardCompatibilityTests extends ElasticsearchTestCase {
|
||||
|
||||
public void testBackwardCompatibility() throws Exception {
|
||||
InternalNode node = new InternalNode();
|
||||
try {
|
||||
try (BufferedReader reader = new BufferedReader(new InputStreamReader(RoutingBackwardCompatibilityTests.class.getResourceAsStream("/org/elasticsearch/cluster/routing/shard_routes.txt"), "UTF-8"))) {
|
||||
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
|
||||
if (line.startsWith("#")) { // comment
|
||||
continue;
|
||||
}
|
||||
String[] parts = line.split("\t");
|
||||
assertEquals(6, parts.length);
|
||||
final String index = parts[0];
|
||||
final int numberOfShards = Integer.parseInt(parts[1]);
|
||||
final String type = parts[2];
|
||||
final String id = parts[3];
|
||||
final String routing = "null".equals(parts[4]) ? null : parts[4];
|
||||
final int expectedShardId = Integer.parseInt(parts[5]);
|
||||
IndexMetaData indexMetaData = IndexMetaData.builder(index).numberOfShards(numberOfShards).numberOfReplicas(randomInt(3)).build();
|
||||
MetaData.Builder metaData = MetaData.builder().put(indexMetaData, false);
|
||||
RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
|
||||
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
|
||||
OperationRouting operationRouting = node.injector().getInstance(OperationRouting.class);
|
||||
assertEquals(expectedShardId, operationRouting.indexShards(clusterState, index, type, id, routing).shardId().getId());
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
node.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,217 @@
|
||||
# Index num_shards _type _id _routing shard_id
|
||||
index 2 type1 foo null 1
|
||||
index 2 type1 foo 42 1
|
||||
index 2 type1 foo my_routing_key 0
|
||||
index 2 type1 bar null 0
|
||||
index 2 type1 bar 42 1
|
||||
index 2 type1 bar my_routing_key 0
|
||||
index 2 type1 foobar null 0
|
||||
index 2 type1 foobar 42 1
|
||||
index 2 type1 foobar my_routing_key 0
|
||||
index 2 type1 elasticsearch null 0
|
||||
index 2 type1 elasticsearch 42 1
|
||||
index 2 type1 elasticsearch my_routing_key 0
|
||||
index 2 type1 0956317778 null 0
|
||||
index 2 type1 0956317778 42 1
|
||||
index 2 type1 0956317778 my_routing_key 0
|
||||
index 2 type1 0 null 1
|
||||
index 2 type1 0 42 1
|
||||
index 2 type1 0 my_routing_key 0
|
||||
index 2 type2 foo null 1
|
||||
index 2 type2 foo 42 1
|
||||
index 2 type2 foo my_routing_key 0
|
||||
index 2 type2 bar null 0
|
||||
index 2 type2 bar 42 1
|
||||
index 2 type2 bar my_routing_key 0
|
||||
index 2 type2 foobar null 0
|
||||
index 2 type2 foobar 42 1
|
||||
index 2 type2 foobar my_routing_key 0
|
||||
index 2 type2 elasticsearch null 0
|
||||
index 2 type2 elasticsearch 42 1
|
||||
index 2 type2 elasticsearch my_routing_key 0
|
||||
index 2 type2 0956317778 null 0
|
||||
index 2 type2 0956317778 42 1
|
||||
index 2 type2 0956317778 my_routing_key 0
|
||||
index 2 type2 0 null 1
|
||||
index 2 type2 0 42 1
|
||||
index 2 type2 0 my_routing_key 0
|
||||
index 5 type1 foo null 4
|
||||
index 5 type1 foo 42 0
|
||||
index 5 type1 foo my_routing_key 3
|
||||
index 5 type1 bar null 4
|
||||
index 5 type1 bar 42 0
|
||||
index 5 type1 bar my_routing_key 3
|
||||
index 5 type1 foobar null 4
|
||||
index 5 type1 foobar 42 0
|
||||
index 5 type1 foobar my_routing_key 3
|
||||
index 5 type1 elasticsearch null 0
|
||||
index 5 type1 elasticsearch 42 0
|
||||
index 5 type1 elasticsearch my_routing_key 3
|
||||
index 5 type1 0956317778 null 3
|
||||
index 5 type1 0956317778 42 0
|
||||
index 5 type1 0956317778 my_routing_key 3
|
||||
index 5 type1 0 null 1
|
||||
index 5 type1 0 42 0
|
||||
index 5 type1 0 my_routing_key 3
|
||||
index 5 type2 foo null 4
|
||||
index 5 type2 foo 42 0
|
||||
index 5 type2 foo my_routing_key 3
|
||||
index 5 type2 bar null 4
|
||||
index 5 type2 bar 42 0
|
||||
index 5 type2 bar my_routing_key 3
|
||||
index 5 type2 foobar null 4
|
||||
index 5 type2 foobar 42 0
|
||||
index 5 type2 foobar my_routing_key 3
|
||||
index 5 type2 elasticsearch null 0
|
||||
index 5 type2 elasticsearch 42 0
|
||||
index 5 type2 elasticsearch my_routing_key 3
|
||||
index 5 type2 0956317778 null 3
|
||||
index 5 type2 0956317778 42 0
|
||||
index 5 type2 0956317778 my_routing_key 3
|
||||
index 5 type2 0 null 1
|
||||
index 5 type2 0 42 0
|
||||
index 5 type2 0 my_routing_key 3
|
||||
index 100 type1 foo null 49
|
||||
index 100 type1 foo 42 75
|
||||
index 100 type1 foo my_routing_key 38
|
||||
index 100 type1 bar null 34
|
||||
index 100 type1 bar 42 75
|
||||
index 100 type1 bar my_routing_key 38
|
||||
index 100 type1 foobar null 74
|
||||
index 100 type1 foobar 42 75
|
||||
index 100 type1 foobar my_routing_key 38
|
||||
index 100 type1 elasticsearch null 20
|
||||
index 100 type1 elasticsearch 42 75
|
||||
index 100 type1 elasticsearch my_routing_key 38
|
||||
index 100 type1 0956317778 null 18
|
||||
index 100 type1 0956317778 42 75
|
||||
index 100 type1 0956317778 my_routing_key 38
|
||||
index 100 type1 0 null 21
|
||||
index 100 type1 0 42 75
|
||||
index 100 type1 0 my_routing_key 38
|
||||
index 100 type2 foo null 49
|
||||
index 100 type2 foo 42 75
|
||||
index 100 type2 foo my_routing_key 38
|
||||
index 100 type2 bar null 34
|
||||
index 100 type2 bar 42 75
|
||||
index 100 type2 bar my_routing_key 38
|
||||
index 100 type2 foobar null 74
|
||||
index 100 type2 foobar 42 75
|
||||
index 100 type2 foobar my_routing_key 38
|
||||
index 100 type2 elasticsearch null 20
|
||||
index 100 type2 elasticsearch 42 75
|
||||
index 100 type2 elasticsearch my_routing_key 38
|
||||
index 100 type2 0956317778 null 18
|
||||
index 100 type2 0956317778 42 75
|
||||
index 100 type2 0956317778 my_routing_key 38
|
||||
index 100 type2 0 null 21
|
||||
index 100 type2 0 42 75
|
||||
index 100 type2 0 my_routing_key 38
|
||||
index2 2 type1 foo null 1
|
||||
index2 2 type1 foo 42 1
|
||||
index2 2 type1 foo my_routing_key 0
|
||||
index2 2 type1 bar null 0
|
||||
index2 2 type1 bar 42 1
|
||||
index2 2 type1 bar my_routing_key 0
|
||||
index2 2 type1 foobar null 0
|
||||
index2 2 type1 foobar 42 1
|
||||
index2 2 type1 foobar my_routing_key 0
|
||||
index2 2 type1 elasticsearch null 0
|
||||
index2 2 type1 elasticsearch 42 1
|
||||
index2 2 type1 elasticsearch my_routing_key 0
|
||||
index2 2 type1 0956317778 null 0
|
||||
index2 2 type1 0956317778 42 1
|
||||
index2 2 type1 0956317778 my_routing_key 0
|
||||
index2 2 type1 0 null 1
|
||||
index2 2 type1 0 42 1
|
||||
index2 2 type1 0 my_routing_key 0
|
||||
index2 2 type2 foo null 1
|
||||
index2 2 type2 foo 42 1
|
||||
index2 2 type2 foo my_routing_key 0
|
||||
index2 2 type2 bar null 0
|
||||
index2 2 type2 bar 42 1
|
||||
index2 2 type2 bar my_routing_key 0
|
||||
index2 2 type2 foobar null 0
|
||||
index2 2 type2 foobar 42 1
|
||||
index2 2 type2 foobar my_routing_key 0
|
||||
index2 2 type2 elasticsearch null 0
|
||||
index2 2 type2 elasticsearch 42 1
|
||||
index2 2 type2 elasticsearch my_routing_key 0
|
||||
index2 2 type2 0956317778 null 0
|
||||
index2 2 type2 0956317778 42 1
|
||||
index2 2 type2 0956317778 my_routing_key 0
|
||||
index2 2 type2 0 null 1
|
||||
index2 2 type2 0 42 1
|
||||
index2 2 type2 0 my_routing_key 0
|
||||
index2 5 type1 foo null 4
|
||||
index2 5 type1 foo 42 0
|
||||
index2 5 type1 foo my_routing_key 3
|
||||
index2 5 type1 bar null 4
|
||||
index2 5 type1 bar 42 0
|
||||
index2 5 type1 bar my_routing_key 3
|
||||
index2 5 type1 foobar null 4
|
||||
index2 5 type1 foobar 42 0
|
||||
index2 5 type1 foobar my_routing_key 3
|
||||
index2 5 type1 elasticsearch null 0
|
||||
index2 5 type1 elasticsearch 42 0
|
||||
index2 5 type1 elasticsearch my_routing_key 3
|
||||
index2 5 type1 0956317778 null 3
|
||||
index2 5 type1 0956317778 42 0
|
||||
index2 5 type1 0956317778 my_routing_key 3
|
||||
index2 5 type1 0 null 1
|
||||
index2 5 type1 0 42 0
|
||||
index2 5 type1 0 my_routing_key 3
|
||||
index2 5 type2 foo null 4
|
||||
index2 5 type2 foo 42 0
|
||||
index2 5 type2 foo my_routing_key 3
|
||||
index2 5 type2 bar null 4
|
||||
index2 5 type2 bar 42 0
|
||||
index2 5 type2 bar my_routing_key 3
|
||||
index2 5 type2 foobar null 4
|
||||
index2 5 type2 foobar 42 0
|
||||
index2 5 type2 foobar my_routing_key 3
|
||||
index2 5 type2 elasticsearch null 0
|
||||
index2 5 type2 elasticsearch 42 0
|
||||
index2 5 type2 elasticsearch my_routing_key 3
|
||||
index2 5 type2 0956317778 null 3
|
||||
index2 5 type2 0956317778 42 0
|
||||
index2 5 type2 0956317778 my_routing_key 3
|
||||
index2 5 type2 0 null 1
|
||||
index2 5 type2 0 42 0
|
||||
index2 5 type2 0 my_routing_key 3
|
||||
index2 100 type1 foo null 49
|
||||
index2 100 type1 foo 42 75
|
||||
index2 100 type1 foo my_routing_key 38
|
||||
index2 100 type1 bar null 34
|
||||
index2 100 type1 bar 42 75
|
||||
index2 100 type1 bar my_routing_key 38
|
||||
index2 100 type1 foobar null 74
|
||||
index2 100 type1 foobar 42 75
|
||||
index2 100 type1 foobar my_routing_key 38
|
||||
index2 100 type1 elasticsearch null 20
|
||||
index2 100 type1 elasticsearch 42 75
|
||||
index2 100 type1 elasticsearch my_routing_key 38
|
||||
index2 100 type1 0956317778 null 18
|
||||
index2 100 type1 0956317778 42 75
|
||||
index2 100 type1 0956317778 my_routing_key 38
|
||||
index2 100 type1 0 null 21
|
||||
index2 100 type1 0 42 75
|
||||
index2 100 type1 0 my_routing_key 38
|
||||
index2 100 type2 foo null 49
|
||||
index2 100 type2 foo 42 75
|
||||
index2 100 type2 foo my_routing_key 38
|
||||
index2 100 type2 bar null 34
|
||||
index2 100 type2 bar 42 75
|
||||
index2 100 type2 bar my_routing_key 38
|
||||
index2 100 type2 foobar null 74
|
||||
index2 100 type2 foobar 42 75
|
||||
index2 100 type2 foobar my_routing_key 38
|
||||
index2 100 type2 elasticsearch null 20
|
||||
index2 100 type2 elasticsearch 42 75
|
||||
index2 100 type2 elasticsearch my_routing_key 38
|
||||
index2 100 type2 0956317778 null 18
|
||||
index2 100 type2 0956317778 42 75
|
||||
index2 100 type2 0956317778 my_routing_key 38
|
||||
index2 100 type2 0 null 21
|
||||
index2 100 type2 0 42 75
|
||||
index2 100 type2 0 my_routing_key 38
|
||||
Loading…
x
Reference in New Issue
Block a user