honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-20 03:45:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve performance of applyDeletedShards

Browse Source 
This commit addresses a performance issue in
IndicesClusterStateService#applyDeletedShards. Namely, the current
implementation is O(number of indices * number of shards). This is
because of an outer loop over the indices and an inner loop over the
assigned shards, all to check if a shard is in the outer index. Instead,
we can group the shards by index, and then just do a map lookup for each
index.

Testing this on a single-node with 2500 indices, each with 2 shards,
creating an index before this optimization takes 0.90s and after this
optimization takes 0.19s.

Relates #18788
This commit is contained in:
Jason Tedor 2016-06-08 16:08:00 -04:00
parent cd0473159a
commit e9017f619e
1 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -71,7 +71,10 @@ import org.elasticsearch.snapshots.RestoreService;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -79,7 +82,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors;
/** /**
* *
@ -285,19 +288,18 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
if (routingNode == null) { if (routingNode == null) {
return; return;
} }
Set<String> newShardAllocationIds = new HashSet<>();
final Map<Index, Set<String>> shardsByIndex = new HashMap<>();
for (ShardRouting shard : routingNode) {
shardsByIndex.computeIfAbsent(shard.index(), k -> new HashSet<>()).add(shard.allocationId().getId());
}
for (IndexService indexService : indicesService) { for (IndexService indexService : indicesService) {
Index index = indexService.index(); Index index = indexService.index();
IndexMetaData indexMetaData = event.state().metaData().index(index); IndexMetaData indexMetaData = event.state().metaData().index(index);
assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index; assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index;
// now, go over and delete shards that needs to get deleted // now, go over and delete shards that needs to get deleted
newShardAllocationIds.clear(); Set<String> newShardAllocationIds = shardsByIndex.getOrDefault(index, Collections.emptySet());
for (ShardRouting shard : routingNode) {
if (shard.index().equals(index)) {
// use the allocation id and not object so we won't be influence by relocation targets
newShardAllocationIds.add(shard.allocationId().getId());
}
}
for (IndexShard existingShard : indexService) { for (IndexShard existingShard : indexService) {
if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) { if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) {
if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {