honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pr/8871

Browse Source
This commit is contained in:
Ryan Ernst 2015-06-24 08:05:17 -07:00
commit 791d111cc0
374 changed files with 15916 additions and 4413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/pom.xml
View File

@ -713,7 +713,7 @@
</data>
<data>
<src>${project.build.directory}/lib</src>
<includes>lucene*, *log4j*, jna*, spatial4j*, jts*, groovy*, antlr-runtime*, asm*</includes>
<excludes>${project.build.finalName}-shaded.jar,${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar,sigar-*.jar</excludes>
<type>directory</type>
<mapper>
<type>perm</type>
@ -880,16 +880,14 @@
<sources>
<source>
<location>target/lib/</location>
<includes>
<include>lucene*</include>
<include>*log4j*</include>
<include>jna*</include>
<include>spatial4j*</include>
<include>jts*</include>
<include>groovy*</include>
<include>antlr-runtime*</include>
<include>asm*</include>
</includes>
<excludes>
<exclude>${project.build.finalName}-shaded.jar</exclude>
<exclude>${project.build.finalName}-sources.jar</exclude>
<exclude>${project.build.finalName}-tests.jar</exclude>
<exclude>${project.build.finalName}-test-sources.jar</exclude>
<exclude>slf4j-api-*.jar</exclude>
<exclude>sigar-*.jar</exclude>
</excludes>
</source>
<source>
<location>${project.build.directory}/</location>

14
core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.action.admin.cluster.repositories.put;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.master.AcknowledgedRequest;
import org.elasticsearch.common.bytes.BytesReference;
@ -29,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
@ -231,8 +231,8 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque
* @param repositoryDefinition repository definition
*/
public PutRepositoryRequest source(String repositoryDefinition) {
try {
return source(XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse repository source [" + repositoryDefinition + "]", e);
}
@ -255,8 +255,8 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque
* @param repositoryDefinition repository definition
*/
public PutRepositoryRequest source(byte[] repositoryDefinition, int offset, int length) {
try {
return source(XContentFactory.xContent(repositoryDefinition, offset, length).createParser(repositoryDefinition, offset, length).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(repositoryDefinition, offset, length).createParser(repositoryDefinition, offset, length)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse repository source", e);
}
@ -269,8 +269,8 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque
* @param repositoryDefinition repository definition
*/
public PutRepositoryRequest source(BytesReference repositoryDefinition) {
try {
return source(XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(repositoryDefinition).createParser(repositoryDefinition)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse template source", e);
}

3
core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.action.admin.cluster.shards;
import org.elasticsearch.cluster.routing.ImmutableShardRouting;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -71,7 +70,7 @@ public class ClusterSearchShardsGroup implements Streamable, ToXContent {
shardId = in.readVInt();
shards = new ShardRouting[in.readVInt()];
for (int i = 0; i < shards.length; i++) {
shards[i] = ImmutableShardRouting.readShardRoutingEntry(in, index, shardId);
shards[i] = ShardRouting.readShardRoutingEntry(in, index, shardId);
}
}

13
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
@ -402,8 +403,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
*/
public CreateSnapshotRequest source(String source) {
if (hasLength(source)) {
try {
return source(XContentFactory.xContent(source).createParser(source).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
return source(parser.mapOrdered());
} catch (Exception e) {
throw new IllegalArgumentException("failed to parse repository source [" + source + "]", e);
}
@ -431,8 +432,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
*/
public CreateSnapshotRequest source(byte[] source, int offset, int length) {
if (length > 0) {
try {
return source(XContentFactory.xContent(source, offset, length).createParser(source, offset, length).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse repository source", e);
}
@ -447,8 +448,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
* @return this request
*/
public CreateSnapshotRequest source(BytesReference source) {
try {
return source(XContentFactory.xContent(source).createParser(source).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse snapshot source", e);
}

13
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
@ -553,8 +554,8 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
*/
public RestoreSnapshotRequest source(String source) {
if (hasLength(source)) {
try {
return source(XContentFactory.xContent(source).createParser(source).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
return source(parser.mapOrdered());
} catch (Exception e) {
throw new IllegalArgumentException("failed to parse repository source [" + source + "]", e);
}
@ -586,8 +587,8 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
*/
public RestoreSnapshotRequest source(byte[] source, int offset, int length) {
if (length > 0) {
try {
return source(XContentFactory.xContent(source, offset, length).createParser(source, offset, length).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse repository source", e);
}
@ -604,8 +605,8 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
* @return this request
*/
public RestoreSnapshotRequest source(BytesReference source) {
try {
return source(XContentFactory.xContent(source).createParser(source).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse template source", e);
}

13
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
View File

@ -36,12 +36,13 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
import org.elasticsearch.snapshots.SnapshotsService;
import org.elasticsearch.snapshots.SnapshotShardsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
@ -51,13 +52,13 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
public static final String ACTION_NAME = SnapshotsStatusAction.NAME + "[nodes]";
private final SnapshotsService snapshotsService;
private final SnapshotShardsService snapshotShardsService;
@Inject
public TransportNodesSnapshotsStatus(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, SnapshotsService snapshotsService, ActionFilters actionFilters) {
public TransportNodesSnapshotsStatus(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, SnapshotShardsService snapshotShardsService, ActionFilters actionFilters) {
super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
Request.class, NodeRequest.class, ThreadPool.Names.GENERIC);
this.snapshotsService = snapshotsService;
this.snapshotShardsService = snapshotShardsService;
}
@Override
@ -99,12 +100,12 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
try {
String nodeId = clusterService.localNode().id();
for (SnapshotId snapshotId : request.snapshotIds) {
ImmutableMap<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotsService.currentSnapshotShards(snapshotId);
Map<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotShardsService.currentSnapshotShards(snapshotId);
if (shardsStatus == null) {
continue;
}
ImmutableMap.Builder<ShardId, SnapshotIndexShardStatus> shardMapBuilder = ImmutableMap.builder();
for (ImmutableMap.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : shardsStatus.entrySet()) {
for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : shardsStatus.entrySet()) {
SnapshotIndexShardStatus shardStatus;
IndexShardSnapshotStatus.Stage stage = shardEntry.getValue().stage();
if (stage != IndexShardSnapshotStatus.Stage.DONE && stage != IndexShardSnapshotStatus.Stage.FAILURE) {

4
core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
View File

@ -364,8 +364,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
public CreateIndexRequest source(BytesReference source) {
XContentType xContentType = XContentFactory.xContentType(source);
if (xContentType != null) {
try {
source(XContentFactory.xContent(xContentType).createParser(source).mapAndClose());
try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
source(parser.map());
} catch (IOException e) {
throw new ElasticsearchParseException("failed to parse source for create index", e);
}

2
core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java
View File

@ -31,7 +31,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardRoutingEntry;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
public class ShardSegments extends BroadcastShardResponse implements Iterable<Segment> {

2
core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
View File

@ -32,7 +32,7 @@ import org.elasticsearch.index.shard.IndexShard;
import java.io.IOException;
import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardRoutingEntry;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
/**
*/

12
core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
View File

@ -308,8 +308,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
* The template source definition.
*/
public PutIndexTemplateRequest source(String templateSource) {
try {
return source(XContentFactory.xContent(templateSource).createParser(templateSource).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(templateSource).createParser(templateSource)) {
return source(parser.mapOrdered());
} catch (Exception e) {
throw new IllegalArgumentException("failed to parse template source [" + templateSource + "]", e);
}
@ -326,8 +326,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
* The template source definition.
*/
public PutIndexTemplateRequest source(byte[] source, int offset, int length) {
try {
return source(XContentFactory.xContent(source, offset, length).createParser(source, offset, length).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse template source", e);
}
@ -337,8 +337,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
* The template source definition.
*/
public PutIndexTemplateRequest source(BytesReference source) {
try {
return source(XContentFactory.xContent(source).createParser(source).mapOrderedAndClose());
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
return source(parser.mapOrdered());
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse template source", e);
}

2
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
View File

@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import java.io.IOException;
import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardRoutingEntry;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
public class ShardUpgradeStatus extends BroadcastShardResponse {

1
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java
View File

@ -42,7 +42,6 @@ import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;

1
core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
View File

@ -38,7 +38,6 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.IndexShard;

35
core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.action.search;
import com.google.common.collect.Iterators;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
@ -31,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.common.xcontent.XContentFactory;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
/**
@ -43,22 +45,22 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
*/
public static class Item implements Streamable {
private SearchResponse response;
private String failureMessage;
private Throwable throwable;
Item() {
}
public Item(SearchResponse response, String failureMessage) {
public Item(SearchResponse response, Throwable throwable) {
this.response = response;
this.failureMessage = failureMessage;
this.throwable = throwable;
}
/**
* Is it a failed search?
*/
public boolean isFailure() {
return failureMessage != null;
return throwable != null;
}
/**
@ -66,7 +68,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
*/
@Nullable
public String getFailureMessage() {
return failureMessage;
return throwable == null ? null : throwable.getMessage();
}
/**
@ -89,7 +91,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
this.response = new SearchResponse();
response.readFrom(in);
} else {
failureMessage = in.readString();
throwable = in.readThrowable();
}
}
@ -100,9 +102,13 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
response.writeTo(out);
} else {
out.writeBoolean(false);
out.writeString(failureMessage);
out.writeThrowable(throwable);
}
}
public Throwable getFailure() {
return throwable;
}
}
private Item[] items;
@ -150,7 +156,19 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
for (Item item : items) {
if (item.isFailure()) {
builder.startObject();
builder.field(Fields.ERROR, item.getFailureMessage());
builder.startObject(Fields.ERROR);
final Throwable t = item.getFailure();
final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(t);
builder.field(Fields.ROOT_CAUSE);
builder.startArray();
for (ElasticsearchException rootCause : rootCauses){
builder.startObject();
rootCause.toXContent(builder, new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE, "true"), params));
builder.endObject();
}
builder.endArray();
ElasticsearchException.toXContent(builder, params, t);
builder.endObject();
builder.endObject();
} else {
builder.startObject();
@ -165,6 +183,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
static final class Fields {
static final XContentBuilderString RESPONSES = new XContentBuilderString("responses");
static final XContentBuilderString ERROR = new XContentBuilderString("error");
static final XContentBuilderString ROOT_CAUSE = new XContentBuilderString("root_cause");
}
@Override

2
core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
View File

@ -69,7 +69,7 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
@Override
public void onFailure(Throwable e) {
responses.set(index, new MultiSearchResponse.Item(null, ExceptionsHelper.detailedMessage(e)));
responses.set(index, new MultiSearchResponse.Item(null, e));
if (counter.decrementAndGet() == 0) {
finishHim();
}

1
core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
View File

@ -43,7 +43,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.settings.Settings;

17
core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@ -177,6 +178,22 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
}
/**
* Checks if all shards in the list have completed
*
* @param shards list of shard statuses
* @return true if all shards have completed (either successfully or failed), false otherwise
*/
public static boolean completed(Collection<ShardSnapshotStatus> shards) {
for (ShardSnapshotStatus status : shards) {
if (status.state().completed() == false) {
return false;
}
}
return true;
}
public static class ShardSnapshotStatus {
private State state;
private String nodeId;

4
core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
View File

@ -45,7 +45,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardRoutingEntry;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
/**
*
@ -183,7 +183,7 @@ public class ShardStateAction extends AbstractComponent {
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
if (oldState != newState && newState.getRoutingNodes().hasUnassigned()) {
logger.trace("unassigned shards after shard failures. scheduling a reroute.");
routingService.scheduleReroute();
routingService.reroute("unassigned shards after shard failures, scheduling a reroute");
}
}
});

5
core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
View File

@ -81,9 +81,8 @@ public class AliasValidator extends AbstractComponent {
public void validateAliasStandalone(Alias alias) {
validateAliasStandalone(alias.name(), alias.indexRouting());
if (Strings.hasLength(alias.filter())) {
try {
XContentParser parser = XContentFactory.xContent(alias.filter()).createParser(alias.filter());
parser.mapAndClose();
try (XContentParser parser = XContentFactory.xContent(alias.filter()).createParser(alias.filter())) {
parser.map();
} catch (Throwable e) {
throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
}

7
core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View File

@ -343,9 +343,10 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
builder.startArray("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
byte[] data = cursor.value.uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> mapping = parser.mapOrderedAndClose();
builder.map(mapping);
try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
Map<String, Object> mapping = parser.mapOrdered();
builder.map(mapping);
}
}
builder.endArray();
}

5
core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -288,7 +288,10 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
public MappingMetaData(CompressedXContent mapping) throws IOException {
this.source = mapping;
Map<String, Object> mappingMap = XContentHelper.createParser(mapping.compressedReference()).mapOrderedAndClose();
Map<String, Object> mappingMap;
try (XContentParser parser = XContentHelper.createParser(mapping.compressedReference())) {
mappingMap = parser.mapOrdered();
}
if (mappingMap.size() != 1) {
throw new IllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
}

6
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
@ -53,6 +54,7 @@ import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper;
@ -451,7 +453,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
}
private Map<String, Object> parseMapping(String mappingSource) throws Exception {
return XContentFactory.xContent(mappingSource).createParser(mappingSource).mapAndClose();
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
return parser.map();
}
}
private void addMappings(Map<String, Map<String, Object>> mappings, Path mappingsDir) throws IOException {

1
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View File

@ -161,6 +161,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
/** All known time settings for an index. */
public static final Set<String> INDEX_TIME_SETTINGS = ImmutableSet.of(
"index.gateway.wait_for_mapping_update_post_recovery",
"index.shard.wait_for_mapping_update_post_recovery",
"index.gc_deletes",
"index.indexing.slowlog.threshold.index.debug",
"index.indexing.slowlog.threshold.index.info",

396
core/src/main/java/org/elasticsearch/cluster/routing/ImmutableShardRouting.java
View File

@ -1,396 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.routing;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.io.Serializable;
/**
* {@link ImmutableShardRouting} immutably encapsulates information about shard
* routings like id, state, version, etc.
*/
public class ImmutableShardRouting implements Streamable, Serializable, ShardRouting {
protected String index;
protected int shardId;
protected String currentNodeId;
protected String relocatingNodeId;
protected boolean primary;
protected ShardRoutingState state;
protected long version;
private transient ShardId shardIdentifier;
protected RestoreSource restoreSource;
protected UnassignedInfo unassignedInfo;
private final transient ImmutableList<ShardRouting> asList;
ImmutableShardRouting() {
this.asList = ImmutableList.of((ShardRouting) this);
}
public ImmutableShardRouting(ShardRouting copy) {
this(copy, copy.version());
}
public ImmutableShardRouting(ShardRouting copy, long version) {
this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), version, copy.unassignedInfo());
}
public ImmutableShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, null, primary, state, version);
}
public ImmutableShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version);
}
public ImmutableShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, null);
}
public ImmutableShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version,
UnassignedInfo unassignedInfo) {
this.index = index;
this.shardId = shardId;
this.currentNodeId = currentNodeId;
this.relocatingNodeId = relocatingNodeId;
this.primary = primary;
this.state = state;
this.asList = ImmutableList.of((ShardRouting) this);
this.version = version;
this.restoreSource = restoreSource;
this.unassignedInfo = unassignedInfo;
assert !(state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) : "unassigned shard must be created with meta";
}
@Override
public String index() {
return this.index;
}
@Override
public String getIndex() {
return index();
}
@Override
public int id() {
return this.shardId;
}
@Override
public int getId() {
return id();
}
@Override
public long version() {
return this.version;
}
@Override
public boolean unassigned() {
return state == ShardRoutingState.UNASSIGNED;
}
@Override
public boolean initializing() {
return state == ShardRoutingState.INITIALIZING;
}
@Override
public boolean active() {
return started() || relocating();
}
@Override
public boolean started() {
return state == ShardRoutingState.STARTED;
}
@Override
public boolean relocating() {
return state == ShardRoutingState.RELOCATING;
}
@Override
public boolean assignedToNode() {
return currentNodeId != null;
}
@Override
public String currentNodeId() {
return this.currentNodeId;
}
@Override
public String relocatingNodeId() {
return this.relocatingNodeId;
}
@Override
public ShardRouting targetRoutingIfRelocating() {
if (!relocating()) {
return null;
}
return new ImmutableShardRouting(index, shardId, relocatingNodeId, currentNodeId, primary, ShardRoutingState.INITIALIZING, version);
}
@Override
public RestoreSource restoreSource() {
return restoreSource;
}
@Override
@Nullable
public UnassignedInfo unassignedInfo() {
return unassignedInfo;
}
@Override
public boolean primary() {
return this.primary;
}
@Override
public ShardRoutingState state() {
return this.state;
}
@Override
public ShardId shardId() {
if (shardIdentifier != null) {
return shardIdentifier;
}
shardIdentifier = new ShardId(index, shardId);
return shardIdentifier;
}
@Override
public ShardIterator shardsIt() {
return new PlainShardIterator(shardId(), asList);
}
public static ImmutableShardRouting readShardRoutingEntry(StreamInput in) throws IOException {
ImmutableShardRouting entry = new ImmutableShardRouting();
entry.readFrom(in);
return entry;
}
public static ImmutableShardRouting readShardRoutingEntry(StreamInput in, String index, int shardId) throws IOException {
ImmutableShardRouting entry = new ImmutableShardRouting();
entry.readFrom(in, index, shardId);
return entry;
}
public void readFrom(StreamInput in, String index, int shardId) throws IOException {
this.index = index;
this.shardId = shardId;
readFromThin(in);
}
@Override
public void readFromThin(StreamInput in) throws IOException {
version = in.readLong();
if (in.readBoolean()) {
currentNodeId = in.readString();
}
if (in.readBoolean()) {
relocatingNodeId = in.readString();
}
primary = in.readBoolean();
state = ShardRoutingState.fromValue(in.readByte());
restoreSource = RestoreSource.readOptionalRestoreSource(in);
if (in.readBoolean()) {
unassignedInfo = new UnassignedInfo(in);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
readFrom(in, in.readString(), in.readVInt());
}
/**
* Writes shard information to {@link StreamOutput} without writing index name and shard id
*
* @param out {@link StreamOutput} to write shard information to
* @throws IOException if something happens during write
*/
@Override
public void writeToThin(StreamOutput out) throws IOException {
out.writeLong(version);
if (currentNodeId != null) {
out.writeBoolean(true);
out.writeString(currentNodeId);
} else {
out.writeBoolean(false);
}
if (relocatingNodeId != null) {
out.writeBoolean(true);
out.writeString(relocatingNodeId);
} else {
out.writeBoolean(false);
}
out.writeBoolean(primary);
out.writeByte(state.value());
if (restoreSource != null) {
out.writeBoolean(true);
restoreSource.writeTo(out);
} else {
out.writeBoolean(false);
}
if (unassignedInfo != null) {
out.writeBoolean(true);
unassignedInfo.writeTo(out);
} else {
out.writeBoolean(false);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(index);
out.writeVInt(shardId);
writeToThin(out);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
// we check on instanceof so we also handle the MutableShardRouting case as well
if (o == null || !(o instanceof ImmutableShardRouting)) {
return false;
}
ImmutableShardRouting that = (ImmutableShardRouting) o;
if (primary != that.primary) {
return false;
}
if (shardId != that.shardId) {
return false;
}
if (currentNodeId != null ? !currentNodeId.equals(that.currentNodeId) : that.currentNodeId != null) {
return false;
}
if (index != null ? !index.equals(that.index) : that.index != null) {
return false;
}
if (relocatingNodeId != null ? !relocatingNodeId.equals(that.relocatingNodeId) : that.relocatingNodeId != null) {
return false;
}
if (state != that.state) {
return false;
}
if (restoreSource != null ? !restoreSource.equals(that.restoreSource) : that.restoreSource != null) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = index != null ? index.hashCode() : 0;
result = 31 * result + shardId;
result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0);
result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
result = 31 * result + (primary ? 1 : 0);
result = 31 * result + (state != null ? state.hashCode() : 0);
result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0);
return result;
}
@Override
public String toString() {
return shortSummary();
}
@Override
public String shortSummary() {
StringBuilder sb = new StringBuilder();
sb.append('[').append(index).append(']').append('[').append(shardId).append(']');
sb.append(", node[").append(currentNodeId).append("], ");
if (relocatingNodeId != null) {
sb.append("relocating [").append(relocatingNodeId).append("], ");
}
if (primary) {
sb.append("[P]");
} else {
sb.append("[R]");
}
if (this.restoreSource != null) {
sb.append(", restoring[" + restoreSource + "]");
}
sb.append(", s[").append(state).append("]");
if (this.unassignedInfo != null) {
sb.append(", ").append(unassignedInfo.toString());
}
return sb.toString();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field("state", state())
.field("primary", primary())
.field("node", currentNodeId())
.field("relocating_node", relocatingNodeId())
.field("shard", shardId().id())
.field("index", shardId().index().name());
if (restoreSource() != null) {
builder.field("restore_source");
restoreSource().toXContent(builder, params);
}
if (unassignedInfo != null) {
unassignedInfo.toXContent(builder, params);
}
return builder.endObject();
}
}

28
core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
View File

@ -70,24 +70,21 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
// shards with state set to UNASSIGNED
private final ImmutableOpenIntMap<IndexShardRoutingTable> shards;
private final ImmutableList<ShardRouting> allShards;
private final ImmutableList<ShardRouting> allActiveShards;
private final List<ShardRouting> allActiveShards;
IndexRoutingTable(String index, ImmutableOpenIntMap<IndexShardRoutingTable> shards) {
this.index = index;
this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt());
this.shards = shards;
ImmutableList.Builder<ShardRouting> allShards = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> allActiveShards = ImmutableList.builder();
for (IntObjectCursor<IndexShardRoutingTable> cursor : shards) {
for (ShardRouting shardRouting : cursor.value) {
allShards.add(shardRouting);
shardRouting.freeze();
if (shardRouting.active()) {
allActiveShards.add(shardRouting);
}
}
}
this.allShards = allShards.build();
this.allActiveShards = allActiveShards.build();
}
@ -275,13 +272,6 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
return shards;
}
/**
* Returns an unordered iterator over all shards (including replicas).
*/
public ShardsIterator randomAllShardsIt() {
return new PlainShardsIterator(shuffler.shuffle(allShards));
}
/**
* Returns an unordered iterator over all active shards (including replicas).
*/
@ -443,9 +433,9 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) {
if (asNew && ignoreShards.contains(shardId)) {
// This shards wasn't completely snapshotted - restore it as new shard
indexShardRoutingBuilder.addShard(new ImmutableShardRouting(index, shardId, null, null, null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
indexShardRoutingBuilder.addShard(new ShardRouting(index, shardId, null, null, null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
} else {
indexShardRoutingBuilder.addShard(new ImmutableShardRouting(index, shardId, null, null, i == 0 ? restoreSource : null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
indexShardRoutingBuilder.addShard(new ShardRouting(index, shardId, null, null, i == 0 ? restoreSource : null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
}
}
shards.put(shardId, indexShardRoutingBuilder.build());
@ -463,7 +453,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) {
IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId), asNew ? false : true);
for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) {
indexShardRoutingBuilder.addShard(new ImmutableShardRouting(index, shardId, null, null, null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
indexShardRoutingBuilder.addShard(new ShardRouting(index, shardId, null, null, null, i == 0, ShardRoutingState.UNASSIGNED, 0, unassignedInfo));
}
shards.put(shardId, indexShardRoutingBuilder.build());
}
@ -474,7 +464,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
for (IntCursor cursor : shards.keys()) {
int shardId = cursor.value;
// version 0, will get updated when reroute will happen
ImmutableShardRouting shard = new ImmutableShardRouting(index, shardId, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
ShardRouting shard = new ShardRouting(index, shardId, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
shards.put(shardId,
new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build()
);
@ -493,7 +483,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
// re-add all the current ones
IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId(), indexShard.primaryAllocatedPostApi());
for (ShardRouting shardRouting : indexShard) {
builder.addShard(new ImmutableShardRouting(shardRouting));
builder.addShard(new ShardRouting(shardRouting));
}
// first check if there is one that is not assigned to a node, and remove it
boolean removed = false;
@ -540,9 +530,9 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
public Builder addShard(IndexShardRoutingTable refData, ShardRouting shard) {
IndexShardRoutingTable indexShard = shards.get(shard.id());
if (indexShard == null) {
indexShard = new IndexShardRoutingTable.Builder(refData.shardId(), refData.primaryAllocatedPostApi()).addShard(new ImmutableShardRouting(shard)).build();
indexShard = new IndexShardRoutingTable.Builder(refData.shardId(), refData.primaryAllocatedPostApi()).addShard(new ShardRouting(shard)).build();
} else {
indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(new ImmutableShardRouting(shard)).build();
indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(new ShardRouting(shard)).build();
}
shards.put(indexShard.shardId().id(), indexShard);
return this;

7
core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.cluster.routing;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.collect.UnmodifiableIterator;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.collect.MapBuilder;
@ -141,7 +140,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
if (shards.get(i).version() == highestVersion) {
shardRoutings.add(shards.get(i));
} else {
shardRoutings.add(new ImmutableShardRouting(shards.get(i), highestVersion));
shardRoutings.add(new ShardRouting(shards.get(i), highestVersion));
}
}
return new IndexShardRoutingTable(shardId, ImmutableList.copyOf(shardRoutings), primaryAllocatedPostApi);
@ -559,7 +558,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
this.primaryAllocatedPostApi = primaryAllocatedPostApi;
}
public Builder addShard(ImmutableShardRouting shardEntry) {
public Builder addShard(ShardRouting shardEntry) {
for (ShardRouting shard : shards) {
// don't add two that map to the same node id
// we rely on the fact that a node does not have primary and backup of the same shard
@ -601,7 +600,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
int size = in.readVInt();
for (int i = 0; i < size; i++) {
ImmutableShardRouting shard = ImmutableShardRouting.readShardRoutingEntry(in, index, iShardId);
ShardRouting shard = ShardRouting.readShardRoutingEntry(in, index, iShardId);
builder.addShard(shard);
}

160
core/src/main/java/org/elasticsearch/cluster/routing/MutableShardRouting.java
View File

@ -1,160 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.routing;
/**
* Similar to {@link ImmutableShardRouting} this class keeps metadata of the current shard. But unlike
* {@link ImmutableShardRouting} the information kept in this class can be modified.
* These modifications include changing the primary state, relocating and assigning the shard
* represented by this class
*/
public class MutableShardRouting extends ImmutableShardRouting {
public MutableShardRouting(ShardRouting copy) {
super(copy);
}
public MutableShardRouting(ShardRouting copy, long version) {
super(copy, version);
}
public MutableShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version) {
super(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version);
assert state != ShardRoutingState.UNASSIGNED : "new mutable routing should not be created with UNASSIGNED state, should moveToUnassigned";
}
/**
* Moves the shard to unassigned state.
*/
void moveToUnassigned(UnassignedInfo unassignedInfo) {
version++;
assert state != ShardRoutingState.UNASSIGNED;
state = ShardRoutingState.UNASSIGNED;
currentNodeId = null;
relocatingNodeId = null;
this.unassignedInfo = unassignedInfo;
}
/**
* Assign this shard to a node.
*
* @param nodeId id of the node to assign this shard to
*/
void assignToNode(String nodeId) {
version++;
if (currentNodeId == null) {
assert state == ShardRoutingState.UNASSIGNED;
state = ShardRoutingState.INITIALIZING;
currentNodeId = nodeId;
relocatingNodeId = null;
} else if (state == ShardRoutingState.STARTED) {
state = ShardRoutingState.RELOCATING;
relocatingNodeId = nodeId;
} else if (state == ShardRoutingState.RELOCATING) {
assert nodeId.equals(relocatingNodeId);
}
}
/**
* Relocate the shard to another node.
*
* @param relocatingNodeId id of the node to relocate the shard
*/
void relocate(String relocatingNodeId) {
version++;
assert state == ShardRoutingState.STARTED;
state = ShardRoutingState.RELOCATING;
this.relocatingNodeId = relocatingNodeId;
}
/**
* Cancel relocation of a shard. The shards state must be set
* to <code>RELOCATING</code>.
*/
void cancelRelocation() {
version++;
assert state == ShardRoutingState.RELOCATING;
assert assignedToNode();
assert relocatingNodeId != null;
state = ShardRoutingState.STARTED;
relocatingNodeId = null;
}
/**
* Moves the shard from started to initializing and bumps the version
*/
void reinitializeShard() {
assert state == ShardRoutingState.STARTED;
version++;
state = ShardRoutingState.INITIALIZING;
}
/**
* Set the shards state to <code>STARTED</code>. The shards state must be
* <code>INITIALIZING</code> or <code>RELOCATING</code>. Any relocation will be
* canceled.
*/
void moveToStarted() {
version++;
assert state == ShardRoutingState.INITIALIZING || state == ShardRoutingState.RELOCATING;
relocatingNodeId = null;
restoreSource = null;
state = ShardRoutingState.STARTED;
unassignedInfo = null; // we keep the unassigned data until the shard is started
}
/**
* Make the shard primary unless it's not Primary
* //TODO: doc exception
*/
void moveToPrimary() {
version++;
if (primary) {
throw new IllegalShardRoutingStateException(this, "Already primary, can't move to primary");
}
primary = true;
}
/**
* Set the primary shard to non-primary
*/
void moveFromPrimary() {
version++;
if (!primary) {
throw new IllegalShardRoutingStateException(this, "Not primary, can't move to replica");
}
primary = false;
}
private long hashVersion = version-1;
private int hashCode = 0;
@Override
public int hashCode() {
hashCode = (hashVersion != version ? super.hashCode() : hashCode);
hashVersion = version;
return hashCode;
}
}

38
core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
View File

@ -33,30 +33,30 @@ import static com.google.common.collect.Lists.newArrayList;
* A {@link RoutingNode} represents a cluster node associated with a single {@link DiscoveryNode} including all shards
* that are hosted on that nodes. Each {@link RoutingNode} has a unique node id that can be used to identify the node.
*/
public class RoutingNode implements Iterable<MutableShardRouting> {
public class RoutingNode implements Iterable<ShardRouting> {
private final String nodeId;
private final DiscoveryNode node;
private final List<MutableShardRouting> shards;
private final List<ShardRouting> shards;
public RoutingNode(String nodeId, DiscoveryNode node) {
this(nodeId, node, new ArrayList<MutableShardRouting>());
this(nodeId, node, new ArrayList<ShardRouting>());
}
public RoutingNode(String nodeId, DiscoveryNode node, List<MutableShardRouting> shards) {
public RoutingNode(String nodeId, DiscoveryNode node, List<ShardRouting> shards) {
this.nodeId = nodeId;
this.node = node;
this.shards = shards;
}
@Override
public Iterator<MutableShardRouting> iterator() {
public Iterator<ShardRouting> iterator() {
return Iterators.unmodifiableIterator(shards.iterator());
}
Iterator<MutableShardRouting> mutableIterator() {
Iterator<ShardRouting> mutableIterator() {
return shards.iterator();
}
@ -85,9 +85,9 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
* Add a new shard to this node
* @param shard Shard to crate on this Node
*/
void add(MutableShardRouting shard) {
void add(ShardRouting shard) {
// TODO use Set with ShardIds for faster lookup.
for (MutableShardRouting shardRouting : shards) {
for (ShardRouting shardRouting : shards) {
if (shardRouting.shardId().equals(shard.shardId())) {
throw new IllegalStateException("Trying to add a shard [" + shard.shardId().index().name() + "][" + shard.shardId().id() + "] to a node [" + nodeId + "] where it already exists");
}
@ -102,7 +102,7 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
*/
public int numberOfShardsWithState(ShardRoutingState... states) {
int count = 0;
for (MutableShardRouting shardEntry : this) {
for (ShardRouting shardEntry : this) {
for (ShardRoutingState state : states) {
if (shardEntry.state() == state) {
count++;
@ -117,9 +117,9 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
* @param states set of states which should be listed
* @return List of shards
*/
public List<MutableShardRouting> shardsWithState(ShardRoutingState... states) {
List<MutableShardRouting> shards = newArrayList();
for (MutableShardRouting shardEntry : this) {
public List<ShardRouting> shardsWithState(ShardRoutingState... states) {
List<ShardRouting> shards = newArrayList();
for (ShardRouting shardEntry : this) {
for (ShardRoutingState state : states) {
if (shardEntry.state() == state) {
shards.add(shardEntry);
@ -135,10 +135,10 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
* @param states set of states which should be listed
* @return a list of shards
*/
public List<MutableShardRouting> shardsWithState(String index, ShardRoutingState... states) {
List<MutableShardRouting> shards = newArrayList();
public List<ShardRouting> shardsWithState(String index, ShardRoutingState... states) {
List<ShardRouting> shards = newArrayList();
for (MutableShardRouting shardEntry : this) {
for (ShardRouting shardEntry : this) {
if (!shardEntry.index().equals(index)) {
continue;
}
@ -156,7 +156,7 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
*/
public int numberOfOwningShards() {
int count = 0;
for (MutableShardRouting shardEntry : this) {
for (ShardRouting shardEntry : this) {
if (shardEntry.state() != ShardRoutingState.RELOCATING) {
count++;
}
@ -168,7 +168,7 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
public String prettyPrint() {
StringBuilder sb = new StringBuilder();
sb.append("-----node_id[").append(nodeId).append("][" + (node == null ? "X" : "V") + "]\n");
for (MutableShardRouting entry : shards) {
for (ShardRouting entry : shards) {
sb.append("--------").append(entry.shortSummary()).append('\n');
}
return sb.toString();
@ -190,11 +190,11 @@ public class RoutingNode implements Iterable<MutableShardRouting> {
return sb.toString();
}
public MutableShardRouting get(int i) {
public ShardRouting get(int i) {
return shards.get(i) ;
}
public Collection<MutableShardRouting> copyShards() {
public Collection<ShardRouting> copyShards() {
return new ArrayList<>(shards);
}

146
core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
View File

@ -53,9 +53,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
private final UnassignedShards unassignedShards = new UnassignedShards();
private final List<MutableShardRouting> ignoredUnassignedShards = newArrayList();
private final List<ShardRouting> ignoredUnassignedShards = newArrayList();
private final Map<ShardId, List<MutableShardRouting>> assignedShards = newHashMap();
private final Map<ShardId, List<ShardRouting>> assignedShards = newHashMap();
private final ImmutableOpenMap<String, ClusterState.Custom> customs;
@ -75,10 +75,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
this.routingTable = clusterState.routingTable();
this.customs = clusterState.customs();
Map<String, List<MutableShardRouting>> nodesToShards = newHashMap();
Map<String, List<ShardRouting>> nodesToShards = newHashMap();
// fill in the nodeToShards with the "live" nodes
for (ObjectCursor<DiscoveryNode> cursor : clusterState.nodes().dataNodes().values()) {
nodesToShards.put(cursor.value.id(), new ArrayList<MutableShardRouting>());
nodesToShards.put(cursor.value.id(), new ArrayList<ShardRouting>());
}
// fill in the inverse of node -> shards allocated
@ -91,12 +91,12 @@ public class RoutingNodes implements Iterable<RoutingNode> {
// by the ShardId, as this is common for primary and replicas.
// A replica Set might have one (and not more) replicas with the state of RELOCATING.
if (shard.assignedToNode()) {
List<MutableShardRouting> entries = nodesToShards.get(shard.currentNodeId());
List<ShardRouting> entries = nodesToShards.get(shard.currentNodeId());
if (entries == null) {
entries = newArrayList();
nodesToShards.put(shard.currentNodeId(), entries);
}
MutableShardRouting sr = new MutableShardRouting(shard);
ShardRouting sr = new ShardRouting(shard);
entries.add(sr);
assignedShardsAdd(sr);
if (shard.relocating()) {
@ -108,7 +108,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
// add the counterpart shard with relocatingNodeId reflecting the source from which
// it's relocating from.
sr = new MutableShardRouting(shard.index(), shard.id(), shard.relocatingNodeId(),
sr = new ShardRouting(shard.index(), shard.id(), shard.relocatingNodeId(),
shard.currentNodeId(), shard.restoreSource(), shard.primary(), ShardRoutingState.INITIALIZING, shard.version());
entries.add(sr);
assignedShardsAdd(sr);
@ -119,14 +119,14 @@ public class RoutingNodes implements Iterable<RoutingNode> {
inactiveShardCount++;
}
} else {
MutableShardRouting sr = new MutableShardRouting(shard);
ShardRouting sr = new ShardRouting(shard);
assignedShardsAdd(sr);
unassignedShards.add(sr);
}
}
}
}
for (Map.Entry<String, List<MutableShardRouting>> entry : nodesToShards.entrySet()) {
for (Map.Entry<String, List<ShardRouting>> entry : nodesToShards.entrySet()) {
String nodeId = entry.getKey();
this.nodesToShards.put(nodeId, new RoutingNode(nodeId, clusterState.nodes().get(nodeId), entry.getValue()));
}
@ -185,7 +185,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return !unassignedShards.isEmpty();
}
public List<MutableShardRouting> ignoredUnassigned() {
public List<ShardRouting> ignoredUnassigned() {
return this.ignoredUnassignedShards;
}
@ -258,8 +258,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* Returns the active primary shard for the given ShardRouting or <code>null</code> if
* no primary is found or the primary is not active.
*/
public MutableShardRouting activePrimary(ShardRouting shard) {
for (MutableShardRouting shardRouting : assignedShards(shard.shardId())) {
public ShardRouting activePrimary(ShardRouting shard) {
for (ShardRouting shardRouting : assignedShards(shard.shardId())) {
if (shardRouting.primary() && shardRouting.active()) {
return shardRouting;
}
@ -271,8 +271,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* Returns one active replica shard for the given ShardRouting shard ID or <code>null</code> if
* no active replica is found.
*/
public MutableShardRouting activeReplica(ShardRouting shard) {
for (MutableShardRouting shardRouting : assignedShards(shard.shardId())) {
public ShardRouting activeReplica(ShardRouting shard) {
for (ShardRouting shardRouting : assignedShards(shard.shardId())) {
if (!shardRouting.primary() && shardRouting.active()) {
return shardRouting;
}
@ -284,7 +284,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* Returns all shards that are not in the state UNASSIGNED with the same shard
* ID as the given shard.
*/
public Iterable<MutableShardRouting> assignedShards(ShardRouting shard) {
public Iterable<ShardRouting> assignedShards(ShardRouting shard) {
return assignedShards(shard.shardId());
}
@ -292,11 +292,11 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* Returns <code>true</code> iff all replicas are active for the given shard routing. Otherwise <code>false</code>
*/
public boolean allReplicasActive(ShardRouting shardRouting) {
final List<MutableShardRouting> shards = assignedShards(shardRouting.shardId());
final List<ShardRouting> shards = assignedShards(shardRouting.shardId());
if (shards.isEmpty() || shards.size() < this.routingTable.index(shardRouting.index()).shard(shardRouting.id()).size()) {
return false; // if we are empty nothing is active if we have less than total at least one is unassigned
}
for (MutableShardRouting shard : shards) {
for (ShardRouting shard : shards) {
if (!shard.active()) {
return false;
}
@ -304,10 +304,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return true;
}
public List<MutableShardRouting> shards(Predicate<MutableShardRouting> predicate) {
List<MutableShardRouting> shards = newArrayList();
public List<ShardRouting> shards(Predicate<ShardRouting> predicate) {
List<ShardRouting> shards = newArrayList();
for (RoutingNode routingNode : this) {
for (MutableShardRouting shardRouting : routingNode) {
for (ShardRouting shardRouting : routingNode) {
if (predicate.apply(shardRouting)) {
shards.add(shardRouting);
}
@ -316,9 +316,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return shards;
}
public List<MutableShardRouting> shardsWithState(ShardRoutingState... state) {
public List<ShardRouting> shardsWithState(ShardRoutingState... state) {
// TODO these are used on tests only - move into utils class
List<MutableShardRouting> shards = newArrayList();
List<ShardRouting> shards = newArrayList();
for (RoutingNode routingNode : this) {
shards.addAll(routingNode.shardsWithState(state));
}
@ -331,15 +331,15 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return shards;
}
public List<MutableShardRouting> shardsWithState(String index, ShardRoutingState... state) {
public List<ShardRouting> shardsWithState(String index, ShardRoutingState... state) {
// TODO these are used on tests only - move into utils class
List<MutableShardRouting> shards = newArrayList();
List<ShardRouting> shards = newArrayList();
for (RoutingNode routingNode : this) {
shards.addAll(routingNode.shardsWithState(index, state));
}
for (ShardRoutingState s : state) {
if (s == ShardRoutingState.UNASSIGNED) {
for (MutableShardRouting unassignedShard : unassignedShards) {
for (ShardRouting unassignedShard : unassignedShards) {
if (unassignedShard.index().equals(index)) {
shards.add(unassignedShard);
}
@ -356,7 +356,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
sb.append(routingNode.prettyPrint());
}
sb.append("---- unassigned\n");
for (MutableShardRouting shardEntry : unassignedShards) {
for (ShardRouting shardEntry : unassignedShards) {
sb.append("--------").append(shardEntry.shortSummary()).append('\n');
}
return sb.toString();
@ -379,7 +379,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* @param shard the shard to be assigned
* @param nodeId the nodeId this shard should initialize on or relocate from
*/
public void assign(MutableShardRouting shard, String nodeId) {
public void assign(ShardRouting shard, String nodeId) {
// state will not change if the shard is already initializing.
ShardRoutingState oldState = shard.state();
shard.assignToNode(nodeId);
@ -400,7 +400,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
/**
* Relocate a shard to another node.
*/
public void relocate(MutableShardRouting shard, String nodeId) {
public void relocate(ShardRouting shard, String nodeId) {
relocatingShards++;
shard.relocate(nodeId);
}
@ -408,7 +408,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
/**
* Mark a shard as started and adjusts internal statistics.
*/
public void started(MutableShardRouting shard) {
public void started(ShardRouting shard) {
if (!shard.active() && shard.relocatingNodeId() == null) {
inactiveShardCount--;
if (shard.primary()) {
@ -424,7 +424,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
/**
* Cancels a relocation of a shard that shard must relocating.
*/
public void cancelRelocation(MutableShardRouting shard) {
public void cancelRelocation(ShardRouting shard) {
relocatingShards--;
shard.cancelRelocation();
}
@ -434,8 +434,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
*
* @param shards the shard to have its primary status swapped.
*/
public void swapPrimaryFlag(MutableShardRouting... shards) {
for (MutableShardRouting shard : shards) {
public void swapPrimaryFlag(ShardRouting... shards) {
for (ShardRouting shard : shards) {
if (shard.primary()) {
shard.moveFromPrimary();
if (shard.unassigned()) {
@ -450,10 +450,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
}
private static final List<MutableShardRouting> EMPTY = Collections.emptyList();
private static final List<ShardRouting> EMPTY = Collections.emptyList();
private List<MutableShardRouting> assignedShards(ShardId shardId) {
final List<MutableShardRouting> replicaSet = assignedShards.get(shardId);
private List<ShardRouting> assignedShards(ShardId shardId) {
final List<ShardRouting> replicaSet = assignedShards.get(shardId);
return replicaSet == null ? EMPTY : Collections.unmodifiableList(replicaSet);
}
@ -462,7 +462,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* the relocation if the shard is relocating.
* @param shard
*/
private void remove(MutableShardRouting shard) {
private void remove(ShardRouting shard) {
if (!shard.active() && shard.relocatingNodeId() == null) {
inactiveShardCount--;
assert inactiveShardCount >= 0;
@ -475,12 +475,12 @@ public class RoutingNodes implements Iterable<RoutingNode> {
assignedShardsRemove(shard);
}
private void assignedShardsAdd(MutableShardRouting shard) {
private void assignedShardsAdd(ShardRouting shard) {
if (shard.unassigned()) {
// no unassigned
return;
}
List<MutableShardRouting> shards = assignedShards.get(shard.shardId());
List<ShardRouting> shards = assignedShards.get(shard.shardId());
if (shards == null) {
shards = Lists.newArrayList();
assignedShards.put(shard.shardId(), shards);
@ -489,17 +489,17 @@ public class RoutingNodes implements Iterable<RoutingNode> {
shards.add(shard);
}
private boolean assertInstanceNotInList(MutableShardRouting shard, List<MutableShardRouting> shards) {
for (MutableShardRouting s : shards) {
private boolean assertInstanceNotInList(ShardRouting shard, List<ShardRouting> shards) {
for (ShardRouting s : shards) {
assert s != shard;
}
return true;
}
private void assignedShardsRemove(MutableShardRouting shard) {
final List<MutableShardRouting> replicaSet = assignedShards.get(shard.shardId());
private void assignedShardsRemove(ShardRouting shard) {
final List<ShardRouting> replicaSet = assignedShards.get(shard.shardId());
if (replicaSet != null) {
final Iterator<MutableShardRouting> iterator = replicaSet.iterator();
final Iterator<ShardRouting> iterator = replicaSet.iterator();
while(iterator.hasNext()) {
// yes we check identity here
if (shard == iterator.next()) {
@ -532,7 +532,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return nodesToShards.values().toArray(new RoutingNode[nodesToShards.size()]);
}
public void reinitShadowPrimary(MutableShardRouting candidate) {
public void reinitShadowPrimary(ShardRouting candidate) {
if (candidate.relocating()) {
cancelRelocation(candidate);
}
@ -542,9 +542,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
public final static class UnassignedShards implements Iterable<MutableShardRouting> {
public final static class UnassignedShards implements Iterable<ShardRouting> {
private final List<MutableShardRouting> unassigned;
private final List<ShardRouting> unassigned;
private int primaries = 0;
private long transactionId = 0;
@ -564,16 +564,16 @@ public class RoutingNodes implements Iterable<RoutingNode> {
sourceTransactionId = -1;
}
public void add(MutableShardRouting mutableShardRouting) {
if(mutableShardRouting.primary()) {
public void add(ShardRouting shardRouting) {
if(shardRouting.primary()) {
primaries++;
}
unassigned.add(mutableShardRouting);
unassigned.add(shardRouting);
transactionId++;
}
public void addAll(Collection<MutableShardRouting> mutableShardRoutings) {
for (MutableShardRouting r : mutableShardRoutings) {
public void addAll(Collection<ShardRouting> mutableShardRoutings) {
for (ShardRouting r : mutableShardRoutings) {
add(r);
}
}
@ -587,17 +587,17 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
@Override
public Iterator<MutableShardRouting> iterator() {
final Iterator<MutableShardRouting> iterator = unassigned.iterator();
return new Iterator<MutableShardRouting>() {
private MutableShardRouting current;
public Iterator<ShardRouting> iterator() {
final Iterator<ShardRouting> iterator = unassigned.iterator();
return new Iterator<ShardRouting>() {
private ShardRouting current;
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public MutableShardRouting next() {
public ShardRouting next() {
return current = iterator.next();
}
@ -639,8 +639,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
return new UnassignedShards(this);
}
public MutableShardRouting[] drain() {
MutableShardRouting[] mutableShardRoutings = unassigned.toArray(new MutableShardRouting[unassigned.size()]);
public ShardRouting[] drain() {
ShardRouting[] mutableShardRoutings = unassigned.toArray(new ShardRouting[unassigned.size()]);
unassigned.clear();
primaries = 0;
transactionId++;
@ -650,7 +650,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
/**
* Calculates RoutingNodes statistics by iterating over all {@link MutableShardRouting}s
* Calculates RoutingNodes statistics by iterating over all {@link ShardRouting}s
* in the cluster to ensure the book-keeping is correct.
* For performance reasons, this should only be called from asserts
*
@ -670,7 +670,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
final Set<ShardId> seenShards = newHashSet();
Map<String, Integer> indicesAndShards = new HashMap<>();
for (RoutingNode node : routingNodes) {
for (MutableShardRouting shard : node) {
for (ShardRouting shard : node) {
if (!shard.active() && shard.relocatingNodeId() == null) {
if (!shard.relocating()) {
inactiveShardCount++;
@ -692,20 +692,20 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
// Assert that the active shard routing are identical.
Set<Map.Entry<String, Integer>> entries = indicesAndShards.entrySet();
final List<MutableShardRouting> shards = newArrayList();
final List<ShardRouting> shards = newArrayList();
for (Map.Entry<String, Integer> e : entries) {
String index = e.getKey();
for (int i = 0; i < e.getValue(); i++) {
for (RoutingNode routingNode : routingNodes) {
for (MutableShardRouting shardRouting : routingNode) {
for (ShardRouting shardRouting : routingNode) {
if (shardRouting.index().equals(index) && shardRouting.id() == i) {
shards.add(shardRouting);
}
}
}
List<MutableShardRouting> mutableShardRoutings = routingNodes.assignedShards(new ShardId(index, i));
List<ShardRouting> mutableShardRoutings = routingNodes.assignedShards(new ShardId(index, i));
assert mutableShardRoutings.size() == shards.size();
for (MutableShardRouting r : mutableShardRoutings) {
for (ShardRouting r : mutableShardRoutings) {
assert shards.contains(r);
shards.remove(r);
}
@ -713,7 +713,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
}
for (MutableShardRouting shard : routingNodes.unassigned()) {
for (ShardRouting shard : routingNodes.unassigned()) {
if (shard.primary()) {
unassignedPrimaryCount++;
}
@ -731,7 +731,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
public class RoutingNodesIterator implements Iterator<RoutingNode>, Iterable<MutableShardRouting> {
public class RoutingNodesIterator implements Iterator<RoutingNode>, Iterable<ShardRouting> {
private RoutingNode current;
private final Iterator<RoutingNode> delegate;
@ -759,15 +759,15 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
@Override
public Iterator<MutableShardRouting> iterator() {
public Iterator<ShardRouting> iterator() {
return nodeShards();
}
}
public final class RoutingNodeIterator implements Iterator<MutableShardRouting>, Iterable<MutableShardRouting> {
public final class RoutingNodeIterator implements Iterator<ShardRouting>, Iterable<ShardRouting> {
private final RoutingNode iterable;
private MutableShardRouting shard;
private final Iterator<MutableShardRouting> delegate;
private ShardRouting shard;
private final Iterator<ShardRouting> delegate;
public RoutingNodeIterator(RoutingNode iterable) {
this.delegate = iterable.mutableIterator();
@ -780,7 +780,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
@Override
public MutableShardRouting next() {
public ShardRouting next() {
return shard = delegate.next();
}
@ -791,13 +791,13 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
@Override
public Iterator<MutableShardRouting> iterator() {
public Iterator<ShardRouting> iterator() {
return iterable.iterator();
}
public void moveToUnassigned(UnassignedInfo unassignedInfo) {
remove();
MutableShardRouting unassigned = new MutableShardRouting(shard); // protective copy of the mutable shard
ShardRouting unassigned = new ShardRouting(shard); // protective copy of the mutable shard
unassigned.moveToUnassigned(unassignedInfo);
unassigned().add(unassigned);
}

99
core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.cluster.routing;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.Priority;
@ -32,12 +31,9 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
/**
* A {@link RoutingService} listens to clusters state. When this service
* receives a {@link ClusterChangedEvent} the cluster state will be verified and
@ -52,21 +48,13 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
*/
public class RoutingService extends AbstractLifecycleComponent<RoutingService> implements ClusterStateListener {
private static final String CLUSTER_UPDATE_TASK_SOURCE = "routing-table-updater";
private final ThreadPool threadPool;
private static final String CLUSTER_UPDATE_TASK_SOURCE = "cluster_reroute";
final ThreadPool threadPool;
private final ClusterService clusterService;
private final AllocationService allocationService;
private final TimeValue schedule;
private volatile boolean routingTableDirty = false;
private volatile Future scheduledRoutingTableFuture;
private AtomicBoolean rerouting = new AtomicBoolean();
private volatile long registeredNextDelaySetting = Long.MAX_VALUE;
private volatile ScheduledFuture registeredNextDelayFuture;
@ -76,8 +64,9 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
this.threadPool = threadPool;
this.clusterService = clusterService;
this.allocationService = allocationService;
this.schedule = settings.getAsTime("cluster.routing.schedule", timeValueSeconds(10));
clusterService.addFirst(this);
if (clusterService != null) {
clusterService.addFirst(this);
}
}
@Override
@ -90,52 +79,27 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
@Override
protected void doClose() {
FutureUtils.cancel(scheduledRoutingTableFuture);
scheduledRoutingTableFuture = null;
FutureUtils.cancel(registeredNextDelayFuture);
clusterService.remove(this);
}
/** make sure that a reroute will be done by the next scheduled check */
public void scheduleReroute() {
routingTableDirty = true;
/**
* Initiates a reroute.
*/
public final void reroute(String reason) {
performReroute(reason);
}
@Override
public void clusterChanged(ClusterChangedEvent event) {
if (event.source().equals(CLUSTER_UPDATE_TASK_SOURCE)) {
if (event.source().startsWith(CLUSTER_UPDATE_TASK_SOURCE)) {
// that's us, ignore this event
return;
}
if (event.state().nodes().localNodeMaster()) {
// we are master, schedule the routing table updater
if (scheduledRoutingTableFuture == null) {
// a new master (us), make sure we reroute shards
routingTableDirty = true;
scheduledRoutingTableFuture = threadPool.scheduleWithFixedDelay(new RoutingTableUpdater(), schedule);
}
if (event.nodesRemoved()) {
// if nodes were removed, we don't want to wait for the scheduled task
// since we want to get primary election as fast as possible
routingTableDirty = true;
reroute();
// Commented out since we make sure to reroute whenever shards changes state or metadata changes state
// } else if (event.routingTableChanged()) {
// routingTableDirty = true;
// reroute();
} else {
if (event.nodesAdded()) {
for (DiscoveryNode node : event.nodesDelta().addedNodes()) {
if (node.dataNode()) {
routingTableDirty = true;
break;
}
}
}
}
// figure out when the next unassigned allocation need to happen from now. If this is larger or equal
// then the last time we checked and scheduled, we are guaranteed to have a reroute until then, so no need
// to schedule again
// figure out when the next unassigned allocation need to happen from now. If this is larger or equal
// then the last time we checked and scheduled, we are guaranteed to have a reroute until then, so no need
// to schedule again
long nextDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSetting(settings, event.state());
if (nextDelaySetting > 0 && nextDelaySetting < registeredNextDelaySetting) {
FutureUtils.cancel(registeredNextDelayFuture);
@ -145,9 +109,8 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
registeredNextDelayFuture = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() {
@Override
protected void doRun() throws Exception {
routingTableDirty = true;
registeredNextDelaySetting = Long.MAX_VALUE;
reroute();
reroute("assign delayed unassigned shards");
}
@Override
@ -158,25 +121,26 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
} else {
logger.trace("no need to schedule reroute due to delayed unassigned, next_delay_setting [{}], registered [{}]", nextDelaySetting, registeredNextDelaySetting);
}
} else {
FutureUtils.cancel(scheduledRoutingTableFuture);
scheduledRoutingTableFuture = null;
}
}
private void reroute() {
// visible for testing
long getRegisteredNextDelaySetting() {
return this.registeredNextDelaySetting;
}
// visible for testing
void performReroute(String reason) {
try {
if (!routingTableDirty) {
return;
}
if (lifecycle.stopped()) {
return;
}
if (rerouting.compareAndSet(false, true) == false) {
logger.trace("already has pending reroute, ignoring");
logger.trace("already has pending reroute, ignoring {}", reason);
return;
}
clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE, Priority.HIGH, new ClusterStateUpdateTask() {
logger.trace("rerouting {}", reason);
clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE + "(" + reason + ")", Priority.HIGH, new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
rerouting.set(false);
@ -205,19 +169,10 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
}
}
});
routingTableDirty = false;
} catch (Throwable e) {
rerouting.set(false);
ClusterState state = clusterService.state();
logger.warn("Failed to reroute routing table, current state:\n{}", e, state.prettyPrint());
}
}
private class RoutingTableUpdater implements Runnable {
@Override
public void run() {
reroute();
logger.warn("failed to reroute routing table, current state:\n{}", e, state.prettyPrint());
}
}
}

4
core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
View File

@ -346,7 +346,7 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
Map<String, IndexRoutingTable.Builder> indexRoutingTableBuilders = newHashMap();
for (RoutingNode routingNode : routingNodes) {
for (MutableShardRouting shardRoutingEntry : routingNode) {
for (ShardRouting shardRoutingEntry : routingNode) {
// every relocating shard has a double entry, ignore the target one.
if (shardRoutingEntry.state() == ShardRoutingState.INITIALIZING && shardRoutingEntry.relocatingNodeId() != null)
continue;
@ -362,7 +362,7 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
indexBuilder.addShard(refData, shardRoutingEntry);
}
}
for (MutableShardRouting shardRoutingEntry : Iterables.concat(routingNodes.unassigned(), routingNodes.ignoredUnassigned())) {
for (ShardRouting shardRoutingEntry : Iterables.concat(routingNodes.unassigned(), routingNodes.ignoredUnassigned())) {
String index = shardRoutingEntry.index();
IndexRoutingTable.Builder indexBuilder = indexRoutingTableBuilders.get(index);
if (indexBuilder == null) {

520
core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
View File

@ -19,78 +19,142 @@
package org.elasticsearch.cluster.routing;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
/**
* Shard routing represents the state of a shard instance allocated in the cluster.
* {@link ShardRouting} immutably encapsulates information about shard
* routings like id, state, version, etc.
*/
public interface ShardRouting extends Streamable, Serializable, ToXContent {
public final class ShardRouting implements Streamable, Serializable, ToXContent {
/**
* The shard id.
*/
ShardId shardId();
protected String index;
protected int shardId;
protected String currentNodeId;
protected String relocatingNodeId;
protected boolean primary;
protected ShardRoutingState state;
protected long version;
private transient ShardId shardIdentifier;
protected RestoreSource restoreSource;
protected UnassignedInfo unassignedInfo;
private final transient List<ShardRouting> asList;
private boolean frozen = false;
ShardRouting() {
this.asList = Arrays.asList(this);
}
public ShardRouting(ShardRouting copy) {
this(copy, copy.version());
}
public ShardRouting(ShardRouting copy, long version) {
this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), version, copy.unassignedInfo());
}
public ShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, null, primary, state, version);
}
public ShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version);
}
public ShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version) {
this(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, null);
}
public ShardRouting(String index, int shardId, String currentNodeId,
String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version,
UnassignedInfo unassignedInfo) {
this.index = index;
this.shardId = shardId;
this.currentNodeId = currentNodeId;
this.relocatingNodeId = relocatingNodeId;
this.primary = primary;
this.state = state;
this.asList = Arrays.asList(this);
this.version = version;
this.restoreSource = restoreSource;
this.unassignedInfo = unassignedInfo;
assert !(state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) : "unassigned shard must be created with meta";
}
/**
* The index name.
*/
String index();
public String index() {
return this.index;
}
/**
* The index name.
*/
String getIndex();
public String getIndex() {
return index();
}
/**
* The shard id.
*/
int id();
public int id() {
return this.shardId;
}
/**
* The shard id.
*/
int getId();
public int getId() {
return id();
}
/**
* The routing version associated with the shard.
*/
long version();
/**
* The shard state.
*/
ShardRoutingState state();
public long version() {
return this.version;
}
/**
* The shard is unassigned (not allocated to any node).
*/
boolean unassigned();
public boolean unassigned() {
return state == ShardRoutingState.UNASSIGNED;
}
/**
* The shard is initializing (usually recovering either from peer shard
* or from gateway).
*/
boolean initializing();
/**
* The shard is in started mode.
*/
boolean started();
/**
* Returns <code>true</code> iff the this shard is currently relocating to
* another node. Otherwise <code>false</code>
*
* @see ShardRoutingState#RELOCATING
*/
boolean relocating();
public boolean initializing() {
return state == ShardRoutingState.INITIALIZING;
}
/**
* Returns <code>true</code> iff the this shard is currently
@ -98,61 +162,427 @@ public interface ShardRouting extends Streamable, Serializable, ToXContent {
* {@link ShardRoutingState#RELOCATING relocating} to another node.
* Otherwise <code>false</code>
*/
boolean active();
public boolean active() {
return started() || relocating();
}
/**
* The shard is in started mode.
*/
public boolean started() {
return state == ShardRoutingState.STARTED;
}
/**
* Returns <code>true</code> iff the this shard is currently relocating to
* another node. Otherwise <code>false</code>
*
* @see ShardRoutingState#RELOCATING
*/
public boolean relocating() {
return state == ShardRoutingState.RELOCATING;
}
/**
* Returns <code>true</code> iff this shard is assigned to a node ie. not
* {@link ShardRoutingState#UNASSIGNED unassigned}. Otherwise <code>false</code>
*/
boolean assignedToNode();
public boolean assignedToNode() {
return currentNodeId != null;
}
/**
* The current node id the shard is allocated on.
*/
String currentNodeId();
public String currentNodeId() {
return this.currentNodeId;
}
/**
* The relocating node id the shard is either relocating to or relocating from.
*/
String relocatingNodeId();
public String relocatingNodeId() {
return this.relocatingNodeId;
}
/**
* If the shard is relocating, return a shard routing representing the target shard or null o.w.
* The target shard routing will be the INITIALIZING state and have relocatingNodeId set to the
* source node.
*/
ShardRouting targetRoutingIfRelocating();
public ShardRouting targetRoutingIfRelocating() {
if (!relocating()) {
return null;
}
return new ShardRouting(index, shardId, relocatingNodeId, currentNodeId, primary, ShardRoutingState.INITIALIZING, version);
}
/**
* Snapshot id and repository where this shard is being restored from
*/
RestoreSource restoreSource();
public RestoreSource restoreSource() {
return restoreSource;
}
/**
* Additional metadata on why the shard is/was unassigned. The metadata is kept around
* until the shard moves to STARTED.
*/
UnassignedInfo unassignedInfo();
@Nullable
public UnassignedInfo unassignedInfo() {
return unassignedInfo;
}
/**
* Returns <code>true</code> iff this shard is a primary.
*/
boolean primary();
public boolean primary() {
return this.primary;
}
/**
* A short description of the shard.
* The shard state.
*/
String shortSummary();
public ShardRoutingState state() {
return this.state;
}
/**
* The shard id.
*/
public ShardId shardId() {
if (shardIdentifier != null) {
return shardIdentifier;
}
shardIdentifier = new ShardId(index, shardId);
return shardIdentifier;
}
/**
* A shard iterator with just this shard in it.
*/
ShardIterator shardsIt();
public ShardIterator shardsIt() {
return new PlainShardIterator(shardId(), asList);
}
public static ShardRouting readShardRoutingEntry(StreamInput in) throws IOException {
ShardRouting entry = new ShardRouting();
entry.readFrom(in);
return entry;
}
public static ShardRouting readShardRoutingEntry(StreamInput in, String index, int shardId) throws IOException {
ShardRouting entry = new ShardRouting();
entry.readFrom(in, index, shardId);
return entry;
}
public void readFrom(StreamInput in, String index, int shardId) throws IOException {
this.index = index;
this.shardId = shardId;
readFromThin(in);
}
public void readFromThin(StreamInput in) throws IOException {
version = in.readLong();
if (in.readBoolean()) {
currentNodeId = in.readString();
}
if (in.readBoolean()) {
relocatingNodeId = in.readString();
}
primary = in.readBoolean();
state = ShardRoutingState.fromValue(in.readByte());
restoreSource = RestoreSource.readOptionalRestoreSource(in);
if (in.readBoolean()) {
unassignedInfo = new UnassignedInfo(in);
}
freeze();
}
@Override
public void readFrom(StreamInput in) throws IOException {
readFrom(in, in.readString(), in.readVInt());
}
/**
* Does not write index name and shard id
* Writes shard information to {@link StreamOutput} without writing index name and shard id
*
* @param out {@link StreamOutput} to write shard information to
* @throws IOException if something happens during write
*/
void writeToThin(StreamOutput out) throws IOException;
public void writeToThin(StreamOutput out) throws IOException {
out.writeLong(version);
if (currentNodeId != null) {
out.writeBoolean(true);
out.writeString(currentNodeId);
} else {
out.writeBoolean(false);
}
void readFromThin(StreamInput in) throws ClassNotFoundException, IOException;
if (relocatingNodeId != null) {
out.writeBoolean(true);
out.writeString(relocatingNodeId);
} else {
out.writeBoolean(false);
}
out.writeBoolean(primary);
out.writeByte(state.value());
if (restoreSource != null) {
out.writeBoolean(true);
restoreSource.writeTo(out);
} else {
out.writeBoolean(false);
}
if (unassignedInfo != null) {
out.writeBoolean(true);
unassignedInfo.writeTo(out);
} else {
out.writeBoolean(false);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(index);
out.writeVInt(shardId);
writeToThin(out);
}
// package private mutators start here
/**
* Moves the shard to unassigned state.
*/
void moveToUnassigned(UnassignedInfo unassignedInfo) {
ensureNotFrozen();
version++;
assert state != ShardRoutingState.UNASSIGNED;
state = ShardRoutingState.UNASSIGNED;
currentNodeId = null;
relocatingNodeId = null;
this.unassignedInfo = unassignedInfo;
}
/**
* Assign this shard to a node.
*
* @param nodeId id of the node to assign this shard to
*/
void assignToNode(String nodeId) {
ensureNotFrozen();
version++;
if (currentNodeId == null) {
assert state == ShardRoutingState.UNASSIGNED;
state = ShardRoutingState.INITIALIZING;
currentNodeId = nodeId;
relocatingNodeId = null;
} else if (state == ShardRoutingState.STARTED) {
state = ShardRoutingState.RELOCATING;
relocatingNodeId = nodeId;
} else if (state == ShardRoutingState.RELOCATING) {
assert nodeId.equals(relocatingNodeId);
}
}
/**
* Relocate the shard to another node.
*
* @param relocatingNodeId id of the node to relocate the shard
*/
void relocate(String relocatingNodeId) {
ensureNotFrozen();
version++;
assert state == ShardRoutingState.STARTED;
state = ShardRoutingState.RELOCATING;
this.relocatingNodeId = relocatingNodeId;
}
/**
* Cancel relocation of a shard. The shards state must be set
* to <code>RELOCATING</code>.
*/
void cancelRelocation() {
ensureNotFrozen();
version++;
assert state == ShardRoutingState.RELOCATING;
assert assignedToNode();
assert relocatingNodeId != null;
state = ShardRoutingState.STARTED;
relocatingNodeId = null;
}
/**
* Moves the shard from started to initializing and bumps the version
*/
void reinitializeShard() {
ensureNotFrozen();
assert state == ShardRoutingState.STARTED;
version++;
state = ShardRoutingState.INITIALIZING;
}
/**
* Set the shards state to <code>STARTED</code>. The shards state must be
* <code>INITIALIZING</code> or <code>RELOCATING</code>. Any relocation will be
* canceled.
*/
void moveToStarted() {
ensureNotFrozen();
version++;
assert state == ShardRoutingState.INITIALIZING || state == ShardRoutingState.RELOCATING;
relocatingNodeId = null;
restoreSource = null;
state = ShardRoutingState.STARTED;
unassignedInfo = null; // we keep the unassigned data until the shard is started
}
/**
* Make the shard primary unless it's not Primary
* //TODO: doc exception
*/
void moveToPrimary() {
ensureNotFrozen();
version++;
if (primary) {
throw new IllegalShardRoutingStateException(this, "Already primary, can't move to primary");
}
primary = true;
}
/**
* Set the primary shard to non-primary
*/
void moveFromPrimary() {
ensureNotFrozen();
version++;
if (!primary) {
throw new IllegalShardRoutingStateException(this, "Not primary, can't move to replica");
}
primary = false;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
// we check on instanceof so we also handle the ImmutableShardRouting case as well
if (o == null || !(o instanceof ShardRouting)) {
return false;
}
ShardRouting that = (ShardRouting) o;
if (primary != that.primary) {
return false;
}
if (shardId != that.shardId) {
return false;
}
if (currentNodeId != null ? !currentNodeId.equals(that.currentNodeId) : that.currentNodeId != null) {
return false;
}
if (index != null ? !index.equals(that.index) : that.index != null) {
return false;
}
if (relocatingNodeId != null ? !relocatingNodeId.equals(that.relocatingNodeId) : that.relocatingNodeId != null) {
return false;
}
if (state != that.state) {
return false;
}
if (restoreSource != null ? !restoreSource.equals(that.restoreSource) : that.restoreSource != null) {
return false;
}
return true;
}
private long hashVersion = version-1;
private int hashCode = 0;
@Override
public int hashCode() {
if (hashVersion == version) {
return hashCode;
}
int result = index != null ? index.hashCode() : 0;
result = 31 * result + shardId;
result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0);
result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
result = 31 * result + (primary ? 1 : 0);
result = 31 * result + (state != null ? state.hashCode() : 0);
result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0);
return hashCode = result;
}
@Override
public String toString() {
return shortSummary();
}
/**
* A short description of the shard.
*/
public String shortSummary() {
StringBuilder sb = new StringBuilder();
sb.append('[').append(index).append(']').append('[').append(shardId).append(']');
sb.append(", node[").append(currentNodeId).append("], ");
if (relocatingNodeId != null) {
sb.append("relocating [").append(relocatingNodeId).append("], ");
}
if (primary) {
sb.append("[P]");
} else {
sb.append("[R]");
}
if (this.restoreSource != null) {
sb.append(", restoring[" + restoreSource + "]");
}
sb.append(", s[").append(state).append("]");
if (this.unassignedInfo != null) {
sb.append(", ").append(unassignedInfo.toString());
}
return sb.toString();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field("state", state())
.field("primary", primary())
.field("node", currentNodeId())
.field("relocating_node", relocatingNodeId())
.field("shard", shardId().id())
.field("index", shardId().index().name());
if (restoreSource() != null) {
builder.field("restore_source");
restoreSource().toXContent(builder, params);
}
if (unassignedInfo != null) {
unassignedInfo.toXContent(builder, params);
}
return builder.endObject();
}
private void ensureNotFrozen() {
if (frozen) {
throw new IllegalStateException("ShardRouting can't be modified anymore - already frozen");
}
}
void freeze() {
frozen = true;
}
boolean isFrozen() {
return frozen;
}
}

38
core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
View File

@ -194,7 +194,7 @@ public class AllocationService extends AbstractComponent {
boolean changed = false;
// create a copy of the shards interleaving between nodes, and check if they can remain
List<MutableShardRouting> shards = new ArrayList<>();
List<ShardRouting> shards = new ArrayList<>();
int index = 0;
boolean found = true;
final RoutingNodes routingNodes = allocation.routingNodes();
@ -210,7 +210,7 @@ public class AllocationService extends AbstractComponent {
index++;
}
for (int i = 0; i < shards.size(); i++) {
MutableShardRouting shardRouting = shards.get(i);
ShardRouting shardRouting = shards.get(i);
// we can only move started shards...
if (!shardRouting.started()) {
continue;
@ -240,9 +240,9 @@ public class AllocationService extends AbstractComponent {
// go over and remove dangling replicas that are initializing for primary shards
List<ShardRouting> shardsToFail = Lists.newArrayList();
for (MutableShardRouting shardEntry : routingNodes.unassigned()) {
for (ShardRouting shardEntry : routingNodes.unassigned()) {
if (shardEntry.primary()) {
for (MutableShardRouting routing : routingNodes.assignedShards(shardEntry)) {
for (ShardRouting routing : routingNodes.assignedShards(shardEntry)) {
if (!routing.primary() && routing.initializing()) {
shardsToFail.add(routing);
}
@ -257,9 +257,9 @@ public class AllocationService extends AbstractComponent {
// now, go over and elect a new primary if possible, not, from this code block on, if one is elected,
// routingNodes.hasUnassignedPrimaries() will potentially be false
for (MutableShardRouting shardEntry : routingNodes.unassigned()) {
for (ShardRouting shardEntry : routingNodes.unassigned()) {
if (shardEntry.primary()) {
MutableShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry);
ShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry);
if (candidate != null) {
IndexMetaData index = allocation.metaData().index(candidate.index());
routingNodes.swapPrimaryFlag(shardEntry, candidate);
@ -268,7 +268,7 @@ public class AllocationService extends AbstractComponent {
// its also relocating, make sure to move the other routing to primary
RoutingNode node = routingNodes.node(candidate.relocatingNodeId());
if (node != null) {
for (MutableShardRouting shardRouting : node) {
for (ShardRouting shardRouting : node) {
if (shardRouting.shardId().equals(candidate.shardId()) && !shardRouting.primary()) {
routingNodes.swapPrimaryFlag(shardRouting);
break;
@ -312,7 +312,7 @@ public class AllocationService extends AbstractComponent {
changed = true;
// now, go over all the shards routing on the node, and fail them
UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]");
for (MutableShardRouting shardRouting : node.copyShards()) {
for (ShardRouting shardRouting : node.copyShards()) {
applyFailedShard(allocation, shardRouting, false, unassignedInfo);
}
// its a dead node, remove it, note, its important to remove it *after* we apply failed shard
@ -333,7 +333,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator currentRoutingNode = routingNodes.routingNodeIter(startedShard.currentNodeId());
if (currentRoutingNode != null) {
for (MutableShardRouting shard : currentRoutingNode) {
for (ShardRouting shard : currentRoutingNode) {
if (shard.shardId().equals(startedShard.shardId())) {
relocatingNodeId = shard.relocatingNodeId();
if (!shard.started()) {
@ -356,7 +356,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator sourceRoutingNode = routingNodes.routingNodeIter(relocatingNodeId);
if (sourceRoutingNode != null) {
while (sourceRoutingNode.hasNext()) {
MutableShardRouting shard = sourceRoutingNode.next();
ShardRouting shard = sourceRoutingNode.next();
if (shard.shardId().equals(startedShard.shardId())) {
if (shard.relocating()) {
dirty = true;
@ -377,7 +377,7 @@ public class AllocationService extends AbstractComponent {
private boolean applyFailedShard(RoutingAllocation allocation, ShardRouting failedShard, boolean addToIgnoreList, UnassignedInfo unassignedInfo) {
// create a copy of the failed shard, since we assume we can change possible references to it without
// changing the state of failed shard
failedShard = new ImmutableShardRouting(failedShard);
failedShard = new ShardRouting(failedShard);
IndexRoutingTable indexRoutingTable = allocation.routingTable().index(failedShard.index());
if (indexRoutingTable == null) {
@ -394,7 +394,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator initializingNode = routingNodes.routingNodeIter(failedShard.currentNodeId());
if (initializingNode != null) {
while (initializingNode.hasNext()) {
MutableShardRouting shardRouting = initializingNode.next();
ShardRouting shardRouting = initializingNode.next();
if (shardRouting.equals(failedShard)) {
dirty = true;
initializingNode.remove();
@ -411,7 +411,7 @@ public class AllocationService extends AbstractComponent {
// now, find the node that we are relocating *from*, and cancel its relocation
RoutingNode relocatingFromNode = routingNodes.node(failedShard.relocatingNodeId());
if (relocatingFromNode != null) {
for (MutableShardRouting shardRouting : relocatingFromNode) {
for (ShardRouting shardRouting : relocatingFromNode) {
if (shardRouting.shardId().equals(failedShard.shardId()) && shardRouting.relocating()) {
dirty = true;
routingNodes.cancelRelocation(shardRouting);
@ -431,7 +431,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator relocatingFromNode = routingNodes.routingNodeIter(failedShard.currentNodeId());
if (relocatingFromNode != null) {
while (relocatingFromNode.hasNext()) {
MutableShardRouting shardRouting = relocatingFromNode.next();
ShardRouting shardRouting = relocatingFromNode.next();
if (shardRouting.equals(failedShard)) {
dirty = true;
if (addToIgnoreList) {
@ -448,7 +448,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator initializingNode = routingNodes.routingNodeIter(failedShard.relocatingNodeId());
if (initializingNode != null) {
while (initializingNode.hasNext()) {
MutableShardRouting shardRouting = initializingNode.next();
ShardRouting shardRouting = initializingNode.next();
if (shardRouting.shardId().equals(failedShard.shardId()) && shardRouting.state() == INITIALIZING) {
dirty = true;
initializingNode.remove();
@ -466,7 +466,7 @@ public class AllocationService extends AbstractComponent {
RoutingNodes.RoutingNodeIterator node = routingNodes.routingNodeIter(failedShard.currentNodeId());
if (node != null) {
while (node.hasNext()) {
MutableShardRouting shardRouting = node.next();
ShardRouting shardRouting = node.next();
if (shardRouting.equals(failedShard)) {
dirty = true;
if (addToIgnoreList) {
@ -477,9 +477,9 @@ public class AllocationService extends AbstractComponent {
// so we give a chance for other allocations and won't create poison failed allocations
// that can keep other shards from being allocated (because of limits applied on how many
// shards we can start per node)
List<MutableShardRouting> shardsToMove = Lists.newArrayList();
for (Iterator<MutableShardRouting> unassignedIt = routingNodes.unassigned().iterator(); unassignedIt.hasNext(); ) {
MutableShardRouting unassignedShardRouting = unassignedIt.next();
List<ShardRouting> shardsToMove = Lists.newArrayList();
for (Iterator<ShardRouting> unassignedIt = routingNodes.unassigned().iterator(); unassignedIt.hasNext(); ) {
ShardRouting unassignedShardRouting = unassignedIt.next();
if (unassignedShardRouting.shardId().equals(failedShard.shardId())) {
unassignedIt.remove();
shardsToMove.add(unassignedShardRouting);

66
core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
View File

@ -23,7 +23,7 @@ import com.google.common.base.Predicate;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntroSorter;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.RoutingNodes;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -118,7 +118,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
}
@Override
public boolean move(MutableShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
return balancer.move(shardRouting, node);
}
@ -227,9 +227,9 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
private final float threshold;
private final MetaData metaData;
private final Predicate<MutableShardRouting> assignedFilter = new Predicate<MutableShardRouting>() {
private final Predicate<ShardRouting> assignedFilter = new Predicate<ShardRouting>() {
@Override
public boolean apply(MutableShardRouting input) {
public boolean apply(ShardRouting input) {
return input.assignedToNode();
}
};
@ -476,7 +476,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
*
* @return <code>true</code> iff the shard has successfully been moved.
*/
public boolean move(MutableShardRouting shard, RoutingNode node ) {
public boolean move(ShardRouting shard, RoutingNode node ) {
if (nodes.isEmpty() || !shard.started()) {
/* with no nodes or a not started shard this is pointless */
return false;
@ -508,7 +508,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
Decision decision = allocation.deciders().canAllocate(shard, target, allocation);
if (decision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
sourceNode.removeShard(shard);
final MutableShardRouting initializingShard = new MutableShardRouting(shard.index(), shard.id(), currentNode.getNodeId(),
final ShardRouting initializingShard = new ShardRouting(shard.index(), shard.id(), currentNode.getNodeId(),
shard.currentNodeId(), shard.restoreSource(), shard.primary(), INITIALIZING, shard.version() + 1);
currentNode.addShard(initializingShard, decision);
routingNodes.assign(initializingShard, target.nodeId());
@ -534,8 +534,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
* on the target node which we respect during the allocation / balancing
* process. In short, this method recreates the status-quo in the cluster.
*/
private void buildModelFromAssigned(Iterable<MutableShardRouting> shards) {
for (MutableShardRouting shard : shards) {
private void buildModelFromAssigned(Iterable<ShardRouting> shards) {
for (ShardRouting shard : shards) {
assert shard.assignedToNode();
/* we skip relocating shards here since we expect an initializing shard with the same id coming in */
if (shard.state() == RELOCATING) {
@ -554,7 +554,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
* Allocates all given shards on the minimal eligable node for the shards index
* with respect to the weight function. All given shards must be unassigned.
*/
private boolean allocateUnassigned(RoutingNodes.UnassignedShards unassigned, List<MutableShardRouting> ignoredUnassigned) {
private boolean allocateUnassigned(RoutingNodes.UnassignedShards unassigned, List<ShardRouting> ignoredUnassigned) {
assert !nodes.isEmpty();
if (logger.isTraceEnabled()) {
logger.trace("Start allocating unassigned shards");
@ -569,10 +569,10 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
* use the sorter to save some iterations.
*/
final AllocationDeciders deciders = allocation.deciders();
final Comparator<MutableShardRouting> comparator = new Comparator<MutableShardRouting>() {
final Comparator<ShardRouting> comparator = new Comparator<ShardRouting>() {
@Override
public int compare(MutableShardRouting o1,
MutableShardRouting o2) {
public int compare(ShardRouting o1,
ShardRouting o2) {
if (o1.primary() ^ o2.primary()) {
return o1.primary() ? -1 : o2.primary() ? 1 : 0;
}
@ -591,15 +591,15 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
* if we allocate for instance (0, R, IDX1) we move the second replica to the secondary array and proceed with
* the next replica. If we could not find a node to allocate (0,R,IDX1) we move all it's replicas to ingoreUnassigned.
*/
MutableShardRouting[] primary = unassigned.drain();
MutableShardRouting[] secondary = new MutableShardRouting[primary.length];
ShardRouting[] primary = unassigned.drain();
ShardRouting[] secondary = new ShardRouting[primary.length];
int secondaryLength = 0;
int primaryLength = primary.length;
ArrayUtil.timSort(primary, comparator);
final Set<ModelNode> throttledNodes = Collections.newSetFromMap(new IdentityHashMap<ModelNode, Boolean>());
do {
for (int i = 0; i < primaryLength; i++) {
MutableShardRouting shard = primary[i];
ShardRouting shard = primary[i];
if (!shard.primary()) {
boolean drop = deciders.canAllocate(shard, allocation).type() == Type.NO;
if (drop) {
@ -717,7 +717,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
}
}
primaryLength = secondaryLength;
MutableShardRouting[] tmp = primary;
ShardRouting[] tmp = primary;
primary = secondary;
secondary = tmp;
secondaryLength = 0;
@ -740,11 +740,11 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
minNode.getNodeId());
}
final RoutingNode node = routingNodes.node(minNode.getNodeId());
MutableShardRouting candidate = null;
ShardRouting candidate = null;
final AllocationDeciders deciders = allocation.deciders();
/* make a copy since we modify this list in the loop */
final ArrayList<MutableShardRouting> shards = new ArrayList<>(index.getAllShards());
for (MutableShardRouting shard : shards) {
final ArrayList<ShardRouting> shards = new ArrayList<>(index.getAllShards());
for (ShardRouting shard : shards) {
if (shard.started()) {
// skip initializing, unassigned and relocating shards we can't relocate them anyway
Decision allocationDecision = deciders.canAllocate(shard, node, allocation);
@ -783,7 +783,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
/* now allocate on the cluster - if we are started we need to relocate the shard */
if (candidate.started()) {
RoutingNode lowRoutingNode = routingNodes.node(minNode.getNodeId());
routingNodes.assign(new MutableShardRouting(candidate.index(), candidate.id(), lowRoutingNode.nodeId(), candidate
routingNodes.assign(new ShardRouting(candidate.index(), candidate.id(), lowRoutingNode.nodeId(), candidate
.currentNodeId(), candidate.restoreSource(), candidate.primary(), INITIALIZING, candidate.version() + 1), lowRoutingNode.nodeId());
routingNodes.relocate(candidate, lowRoutingNode.nodeId());
@ -839,8 +839,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return index == null ? 0 : index.numShards();
}
public Collection<MutableShardRouting> shards() {
Collection<MutableShardRouting> result = new ArrayList<>();
public Collection<ShardRouting> shards() {
Collection<ShardRouting> result = new ArrayList<>();
for (ModelIndex index : indices.values()) {
result.addAll(index.getAllShards());
}
@ -855,7 +855,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return -1;
}
public void addShard(MutableShardRouting shard, Decision decision) {
public void addShard(ShardRouting shard, Decision decision) {
numShards = -1;
ModelIndex index = indices.get(shard.index());
if (index == null) {
@ -865,7 +865,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
index.addShard(shard, decision);
}
public Decision removeShard(MutableShardRouting shard) {
public Decision removeShard(ShardRouting shard) {
numShards = -1;
ModelIndex index = indices.get(shard.index());
Decision removed = null;
@ -890,7 +890,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return indices.values().iterator();
}
public boolean containsShard(MutableShardRouting shard) {
public boolean containsShard(ShardRouting shard) {
ModelIndex index = getIndex(shard.getIndex());
return index == null ? false : index.containsShard(shard);
}
@ -899,7 +899,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
static final class ModelIndex {
private final String id;
private final Map<MutableShardRouting, Decision> shards = new HashMap<>();
private final Map<ShardRouting, Decision> shards = new HashMap<>();
private int numPrimaries = -1;
private int highestPrimary = -1;
@ -910,7 +910,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
public int highestPrimary() {
if (highestPrimary == -1) {
int maxId = -1;
for (MutableShardRouting shard : shards.keySet()) {
for (ShardRouting shard : shards.keySet()) {
if (shard.primary()) {
maxId = Math.max(maxId, shard.id());
}
@ -924,7 +924,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return id;
}
public Decision getDecicion(MutableShardRouting shard) {
public Decision getDecicion(ShardRouting shard) {
return shards.get(shard);
}
@ -932,14 +932,14 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return shards.size();
}
public Collection<MutableShardRouting> getAllShards() {
public Collection<ShardRouting> getAllShards() {
return shards.keySet();
}
public int numPrimaries() {
if (numPrimaries == -1) {
int num = 0;
for (MutableShardRouting shard : shards.keySet()) {
for (ShardRouting shard : shards.keySet()) {
if (shard.primary()) {
num++;
}
@ -949,19 +949,19 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
return numPrimaries;
}
public Decision removeShard(MutableShardRouting shard) {
public Decision removeShard(ShardRouting shard) {
highestPrimary = numPrimaries = -1;
return shards.remove(shard);
}
public void addShard(MutableShardRouting shard, Decision decision) {
public void addShard(ShardRouting shard, Decision decision) {
highestPrimary = numPrimaries = -1;
assert decision != null;
assert !shards.containsKey(shard) : "Shard already allocated on current node: " + shards.get(shard) + " " + shard;
shards.put(shard, decision);
}
public boolean containsShard(MutableShardRouting shard) {
public boolean containsShard(ShardRouting shard) {
return shards.containsKey(shard);
}
}

3
core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing.allocation.allocator;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -74,5 +73,5 @@ public interface ShardsAllocator {
* @param allocation current node allocation
* @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
*/
boolean move(MutableShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation);
boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation);
}

4
core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.cluster.routing.allocation.allocator;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@ -80,7 +80,7 @@ public class ShardsAllocators extends AbstractComponent implements ShardsAllocat
}
@Override
public boolean move(MutableShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
return allocator.move(shardRouting, node, allocation);
}
}

9
core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java
View File

@ -19,10 +19,9 @@
package org.elasticsearch.cluster.routing.allocation.command;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@ -168,8 +167,8 @@ public class AllocateAllocationCommand implements AllocationCommand {
public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) {
DiscoveryNode discoNode = allocation.nodes().resolveNode(node);
MutableShardRouting shardRouting = null;
for (MutableShardRouting routing : allocation.routingNodes().unassigned()) {
ShardRouting shardRouting = null;
for (ShardRouting routing : allocation.routingNodes().unassigned()) {
if (routing.shardId().equals(shardId)) {
// prefer primaries first to allocate
if (shardRouting == null || routing.primary()) {
@ -219,7 +218,7 @@ public class AllocateAllocationCommand implements AllocationCommand {
throw new IllegalArgumentException("[allocate] allocation of " + shardId + " on node " + discoNode + " is not allowed, reason: " + decision);
}
// go over and remove it from the unassigned
for (Iterator<MutableShardRouting> it = allocation.routingNodes().unassigned().iterator(); it.hasNext(); ) {
for (Iterator<ShardRouting> it = allocation.routingNodes().unassigned().iterator(); it.hasNext(); ) {
if (it.next() != shardRouting) {
continue;
}

6
core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
View File

@ -164,7 +164,7 @@ public class CancelAllocationCommand implements AllocationCommand {
DiscoveryNode discoNode = allocation.nodes().resolveNode(node);
boolean found = false;
for (RoutingNodes.RoutingNodeIterator it = allocation.routingNodes().routingNodeIter(discoNode.id()); it.hasNext(); ) {
MutableShardRouting shardRouting = it.next();
ShardRouting shardRouting = it.next();
if (!shardRouting.shardId().equals(shardId)) {
continue;
}
@ -176,7 +176,7 @@ public class CancelAllocationCommand implements AllocationCommand {
// and cancel the relocating state from the shard its being relocated from
RoutingNode relocatingFromNode = allocation.routingNodes().node(shardRouting.relocatingNodeId());
if (relocatingFromNode != null) {
for (MutableShardRouting fromShardRouting : relocatingFromNode) {
for (ShardRouting fromShardRouting : relocatingFromNode) {
if (fromShardRouting.shardId().equals(shardRouting.shardId()) && fromShardRouting.state() == RELOCATING) {
allocation.routingNodes().cancelRelocation(fromShardRouting);
break;
@ -200,7 +200,7 @@ public class CancelAllocationCommand implements AllocationCommand {
RoutingNodes.RoutingNodeIterator initializingNode = allocation.routingNodes().routingNodeIter(shardRouting.relocatingNodeId());
if (initializingNode != null) {
while (initializingNode.hasNext()) {
MutableShardRouting initializingShardRouting = initializingNode.next();
ShardRouting initializingShardRouting = initializingNode.next();
if (initializingShardRouting.shardId().equals(shardRouting.shardId()) && initializingShardRouting.state() == INITIALIZING) {
initializingNode.remove();
}

7
core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java
View File

@ -19,10 +19,9 @@
package org.elasticsearch.cluster.routing.allocation.command;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
@ -152,7 +151,7 @@ public class MoveAllocationCommand implements AllocationCommand {
Decision decision = null;
boolean found = false;
for (MutableShardRouting shardRouting : allocation.routingNodes().node(fromDiscoNode.id())) {
for (ShardRouting shardRouting : allocation.routingNodes().node(fromDiscoNode.id())) {
if (!shardRouting.shardId().equals(shardId)) {
continue;
}
@ -180,7 +179,7 @@ public class MoveAllocationCommand implements AllocationCommand {
// its being throttled, maybe have a flag to take it into account and fail? for now, just do it since the "user" wants it...
}
allocation.routingNodes().assign(new MutableShardRouting(shardRouting.index(), shardRouting.id(),
allocation.routingNodes().assign(new ShardRouting(shardRouting.index(), shardRouting.id(),
toRoutingNode.nodeId(), shardRouting.currentNodeId(), shardRouting.restoreSource(),
shardRouting.primary(), ShardRoutingState.INITIALIZING, shardRouting.version() + 1), toRoutingNode.nodeId());

3
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.cluster.routing.allocation.decider;
import com.carrotsearch.hppc.ObjectIntHashMap;
import com.google.common.collect.Maps;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@ -185,7 +184,7 @@ public class AwarenessAllocationDecider extends AllocationDecider {
// build the count of shards per attribute value
ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
for (MutableShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting)) {
for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting)) {
// if the shard is relocating, then make sure we count it as part of the node it is relocating to
if (assignedShard.relocating()) {
RoutingNode relocationNode = allocation.routingNodes().node(assignedShard.relocatingNodeId());

176
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
View File

@ -295,63 +295,14 @@ public class DiskThresholdDecider extends AllocationDecider {
@Override
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
double usedDiskThresholdLow = 100.0 - DiskThresholdDecider.this.freeDiskThresholdLow;
double usedDiskThresholdHigh = 100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh;
// Always allow allocation if the decider is disabled
if (!enabled) {
return allocation.decision(Decision.YES, NAME, "disk threshold decider disabled");
}
// Allow allocation regardless if only a single node is available
if (allocation.nodes().size() <= 1) {
if (logger.isTraceEnabled()) {
logger.trace("Only a single node is present, allowing allocation");
}
return allocation.decision(Decision.YES, NAME, "only a single node is present");
}
// Fail open there is no info available
ClusterInfo clusterInfo = allocation.clusterInfo();
if (clusterInfo == null) {
if (logger.isTraceEnabled()) {
logger.trace("Cluster info unavailable for disk threshold decider, allowing allocation.");
}
return allocation.decision(Decision.YES, NAME, "cluster info unavailable");
}
// Fail open if there are no disk usages available
Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
Map<String, Long> shardSizes = clusterInfo.getShardSizes();
if (usages.isEmpty()) {
if (logger.isTraceEnabled()) {
logger.trace("Unable to determine disk usages for disk-aware allocation, allowing allocation");
}
return allocation.decision(Decision.YES, NAME, "disk usages unavailable");
}
DiskUsage usage = usages.get(node.nodeId());
if (usage == null) {
// If there is no usage, and we have other nodes in the cluster,
// use the average usage for all nodes as the usage for this node
usage = averageUsage(node, usages);
if (logger.isDebugEnabled()) {
logger.debug("Unable to determine disk usage for [{}], defaulting to average across nodes [{} total] [{} free] [{}% free]",
node.nodeId(), usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeDiskAsPercentage());
}
}
if (includeRelocations) {
long relocatingShardsSize = sizeOfRelocatingShards(node, shardSizes, false);
DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().name(),
usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize);
if (logger.isTraceEnabled()) {
logger.trace("usage without relocations: {}", usage);
logger.trace("usage with relocations: [{} bytes] {}", relocatingShardsSize, usageIncludingRelocations);
}
usage = usageIncludingRelocations;
final Decision decision = earlyTerminate(allocation);
if (decision != null) {
return decision;
}
final double usedDiskThresholdLow = 100.0 - DiskThresholdDecider.this.freeDiskThresholdLow;
final double usedDiskThresholdHigh = 100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh;
DiskUsage usage = getDiskUsage(node, allocation);
// First, check that the node currently over the low watermark
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
// Cache the used disk percentage for displaying disk percentages consistent with documentation
@ -432,9 +383,10 @@ public class DiskThresholdDecider extends AllocationDecider {
}
// Secondly, check that allocating the shard to this node doesn't put it above the high watermark
Map<String, Long> shardSizes = allocation.clusterInfo().getShardSizes();
Long shardSize = shardSizes.get(shardIdentifierFromRouting(shardRouting));
shardSize = shardSize == null ? 0 : shardSize;
double freeSpaceAfterShard = this.freeDiskPercentageAfterShardAssigned(usage, shardSize);
double freeSpaceAfterShard = freeDiskPercentageAfterShardAssigned(usage, shardSize);
long freeBytesAfterShard = freeBytes - shardSize;
if (freeBytesAfterShard < freeBytesThresholdHigh.bytes()) {
logger.warn("After allocating, node [{}] would have less than the required {} free bytes threshold ({} bytes free), preventing allocation",
@ -454,53 +406,11 @@ public class DiskThresholdDecider extends AllocationDecider {
@Override
public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
if (!enabled) {
return allocation.decision(Decision.YES, NAME, "disk threshold decider disabled");
final Decision decision = earlyTerminate(allocation);
if (decision != null) {
return decision;
}
// Allow allocation regardless if only a single node is available
if (allocation.nodes().size() <= 1) {
return allocation.decision(Decision.YES, NAME, "only a single node is present");
}
ClusterInfo clusterInfo = allocation.clusterInfo();
if (clusterInfo == null) {
if (logger.isTraceEnabled()) {
logger.trace("Cluster info unavailable for disk threshold decider, allowing allocation.");
}
return allocation.decision(Decision.YES, NAME, "cluster info unavailable");
}
Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
if (usages.isEmpty()) {
if (logger.isTraceEnabled()) {
logger.trace("Unable to determine disk usages for disk-aware allocation, allowing allocation");
}
return allocation.decision(Decision.YES, NAME, "disk usages unavailable");
}
DiskUsage usage = usages.get(node.nodeId());
if (usage == null) {
// If there is no usage, and we have other nodes in the cluster,
// use the average usage for all nodes as the usage for this node
usage = averageUsage(node, usages);
if (logger.isDebugEnabled()) {
logger.debug("Unable to determine disk usage for {}, defaulting to average across nodes [{} total] [{} free] [{}% free]",
node.nodeId(), usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeDiskAsPercentage());
}
}
if (includeRelocations) {
Map<String, Long> shardSizes = clusterInfo.getShardSizes();
long relocatingShardsSize = sizeOfRelocatingShards(node, shardSizes, true);
DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().name(),
usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize);
if (logger.isTraceEnabled()) {
logger.trace("usage without relocations: {}", usage);
logger.trace("usage with relocations: [{} bytes] {}", relocatingShardsSize, usageIncludingRelocations);
}
usage = usageIncludingRelocations;
}
DiskUsage usage = getDiskUsage(node, allocation);
// If this node is already above the high threshold, the shard cannot remain (get it off!)
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
long freeBytes = usage.getFreeBytes();
@ -527,6 +437,34 @@ public class DiskThresholdDecider extends AllocationDecider {
return allocation.decision(Decision.YES, NAME, "enough disk for shard to remain on node, free: [%s]", new ByteSizeValue(freeBytes));
}
private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation) {
ClusterInfo clusterInfo = allocation.clusterInfo();
Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
DiskUsage usage = usages.get(node.nodeId());
if (usage == null) {
// If there is no usage, and we have other nodes in the cluster,
// use the average usage for all nodes as the usage for this node
usage = averageUsage(node, usages);
if (logger.isDebugEnabled()) {
logger.debug("Unable to determine disk usage for {}, defaulting to average across nodes [{} total] [{} free] [{}% free]",
node.nodeId(), usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeDiskAsPercentage());
}
}
if (includeRelocations) {
Map<String, Long> shardSizes = clusterInfo.getShardSizes();
long relocatingShardsSize = sizeOfRelocatingShards(node, shardSizes, true);
DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().name(),
usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize);
if (logger.isTraceEnabled()) {
logger.trace("usage without relocations: {}", usage);
logger.trace("usage with relocations: [{} bytes] {}", relocatingShardsSize, usageIncludingRelocations);
}
usage = usageIncludingRelocations;
}
return usage;
}
/**
* Returns a {@link DiskUsage} for the {@link RoutingNode} using the
* average usage of other nodes in the disk usage map.
@ -604,4 +542,38 @@ public class DiskThresholdDecider extends AllocationDecider {
}
}
}
private Decision earlyTerminate(RoutingAllocation allocation) {
// Always allow allocation if the decider is disabled
if (!enabled) {
return allocation.decision(Decision.YES, NAME, "disk threshold decider disabled");
}
// Allow allocation regardless if only a single node is available
if (allocation.nodes().size() <= 1) {
if (logger.isTraceEnabled()) {
logger.trace("Only a single node is present, allowing allocation");
}
return allocation.decision(Decision.YES, NAME, "only a single node is present");
}
// Fail open there is no info available
final ClusterInfo clusterInfo = allocation.clusterInfo();
if (clusterInfo == null) {
if (logger.isTraceEnabled()) {
logger.trace("Cluster info unavailable for disk threshold decider, allowing allocation.");
}
return allocation.decision(Decision.YES, NAME, "cluster info unavailable");
}
final Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
// Fail open if there are no disk usages available
if (usages.isEmpty()) {
if (logger.isTraceEnabled()) {
logger.trace("Unable to determine disk usages for disk-aware allocation, allowing allocation");
}
return allocation.decision(Decision.YES, NAME, "disk usages unavailable");
}
return null;
}
}

3
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.RoutingNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
@ -53,7 +52,7 @@ public class NodeVersionAllocationDecider extends AllocationDecider {
// we are the primary we can allocate wherever
return allocation.decision(Decision.YES, NAME, "primary shard can be allocated anywhere");
}
final MutableShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
final ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
if (primary == null) { // we have a primary - it's a start ;)
return allocation.decision(Decision.YES, NAME, "no active primary shard yet");
}

3
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@ -48,7 +47,7 @@ public class ReplicaAfterPrimaryActiveAllocationDecider extends AllocationDecide
if (shardRouting.primary()) {
return allocation.decision(Decision.YES, NAME, "shard is primary");
}
MutableShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
if (primary == null) {
return allocation.decision(Decision.NO, NAME, "primary shard is not yet active");
}

7
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@ -59,8 +58,8 @@ public class SameShardAllocationDecider extends AllocationDecider {
@Override
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
Iterable<MutableShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting);
for (MutableShardRouting assignedShard : assignedShards) {
Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting);
for (ShardRouting assignedShard : assignedShards) {
if (node.nodeId().equals(assignedShard.currentNodeId())) {
return allocation.decision(Decision.NO, NAME, "shard cannot be allocated on same node [%s] it already exists on", node.nodeId());
}
@ -83,7 +82,7 @@ public class SameShardAllocationDecider extends AllocationDecider {
}
}
if (checkNodeOnSameHost) {
for (MutableShardRouting assignedShard : assignedShards) {
for (ShardRouting assignedShard : assignedShards) {
if (checkNode.nodeId().equals(assignedShard.currentNodeId())) {
return allocation.decision(Decision.NO, NAME,
"shard cannot be allocated on same host [%s] it already exists on", node.nodeId());

5
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -71,7 +70,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
}
int nodeCount = 0;
for (MutableShardRouting nodeShard : node) {
for (ShardRouting nodeShard : node) {
if (!nodeShard.index().equals(shardRouting.index())) {
continue;
}
@ -97,7 +96,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
}
int nodeCount = 0;
for (MutableShardRouting nodeShard : node) {
for (ShardRouting nodeShard : node) {
if (!nodeShard.index().equals(shardRouting.index())) {
continue;
}

5
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -81,7 +80,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
// primary is unassigned, means we are going to do recovery from gateway
// count *just the primary* currently doing recovery on the node and check against concurrent_recoveries
int primariesInRecovery = 0;
for (MutableShardRouting shard : node) {
for (ShardRouting shard : node) {
// when a primary shard is INITIALIZING, it can be because of *initial recovery* or *relocation from another node*
// we only count initial recoveries here, so we need to make sure that relocating node is null
if (shard.state() == ShardRoutingState.INITIALIZING && shard.primary() && shard.relocatingNodeId() == null) {
@ -106,7 +105,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
@Override
public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
int currentRecoveries = 0;
for (MutableShardRouting shard : node) {
for (ShardRouting shard : node) {
if (shard.state() == ShardRoutingState.INITIALIZING || shard.state() == ShardRoutingState.RELOCATING) {
currentRecoveries++;
}

16
core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
View File

@ -64,8 +64,8 @@ public class XContentHelper {
public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException {
try {
XContentParser parser;
XContentType contentType;
InputStream input;
Compressor compressor = CompressorFactory.compressor(bytes);
if (compressor != null) {
InputStream compressedStreamInput = compressor.streamInput(bytes.streamInput());
@ -73,15 +73,17 @@ public class XContentHelper {
compressedStreamInput = new BufferedInputStream(compressedStreamInput);
}
contentType = XContentFactory.xContentType(compressedStreamInput);
parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput);
input = compressedStreamInput;
} else {
contentType = XContentFactory.xContentType(bytes);
parser = XContentFactory.xContent(contentType).createParser(bytes.streamInput());
input = bytes.streamInput();
}
if (ordered) {
return Tuple.tuple(contentType, parser.mapOrderedAndClose());
} else {
return Tuple.tuple(contentType, parser.mapAndClose());
try (XContentParser parser = XContentFactory.xContent(contentType).createParser(input)) {
if (ordered) {
return Tuple.tuple(contentType, parser.mapOrdered());
} else {
return Tuple.tuple(contentType, parser.map());
}
}
} catch (IOException e) {
throw new ElasticsearchParseException("Failed to parse content to map", e);

5
core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lease.Releasable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
@ -130,9 +131,9 @@ public interface XContentParser extends Releasable {
Map<String, Object> mapOrdered() throws IOException;
Map<String, Object> mapAndClose() throws IOException;
List<Object> list() throws IOException;
Map<String, Object> mapOrderedAndClose() throws IOException;
List<Object> listOrderedMap() throws IOException;
String text() throws IOException;

40
core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
View File

@ -214,24 +214,15 @@ public abstract class AbstractXContentParser implements XContentParser {
}
@Override
public Map<String, Object> mapAndClose() throws IOException {
try {
return map();
} finally {
close();
}
public List<Object> list() throws IOException {
return readList(this);
}
@Override
public Map<String, Object> mapOrderedAndClose() throws IOException {
try {
return mapOrdered();
} finally {
close();
}
public List<Object> listOrderedMap() throws IOException {
return readListOrderedMap(this);
}
static interface MapFactory {
Map<String, Object> newMap();
}
@ -258,6 +249,14 @@ public abstract class AbstractXContentParser implements XContentParser {
return readMap(parser, ORDERED_MAP_FACTORY);
}
static List<Object> readList(XContentParser parser) throws IOException {
return readList(parser, SIMPLE_MAP_FACTORY);
}
static List<Object> readListOrderedMap(XContentParser parser) throws IOException {
return readList(parser, ORDERED_MAP_FACTORY);
}
static Map<String, Object> readMap(XContentParser parser, MapFactory mapFactory) throws IOException {
Map<String, Object> map = mapFactory.newMap();
XContentParser.Token token = parser.currentToken();
@ -278,15 +277,22 @@ public abstract class AbstractXContentParser implements XContentParser {
return map;
}
private static List<Object> readList(XContentParser parser, MapFactory mapFactory, XContentParser.Token token) throws IOException {
static List<Object> readList(XContentParser parser, MapFactory mapFactory) throws IOException {
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.FIELD_NAME) {
token = parser.nextToken();
}
if (token == XContentParser.Token.START_ARRAY) {
token = parser.nextToken();
}
ArrayList<Object> list = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
for (; token != XContentParser.Token.END_ARRAY; token = parser.nextToken()) {
list.add(readValue(parser, mapFactory, token));
}
return list;
}
private static Object readValue(XContentParser parser, MapFactory mapFactory, XContentParser.Token token) throws IOException {
static Object readValue(XContentParser parser, MapFactory mapFactory, XContentParser.Token token) throws IOException {
if (token == XContentParser.Token.VALUE_NULL) {
return null;
} else if (token == XContentParser.Token.VALUE_STRING) {
@ -307,7 +313,7 @@ public abstract class AbstractXContentParser implements XContentParser {
} else if (token == XContentParser.Token.START_OBJECT) {
return readMap(parser, mapFactory);
} else if (token == XContentParser.Token.START_ARRAY) {
return readList(parser, mapFactory, token);
return readList(parser, mapFactory);
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
return parser.binaryValue();
}

4
core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
View File

@ -176,7 +176,9 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
nodesBuilder.put(discovery.localNode);
}
nodesBuilder.localNodeId(master.localNode().id()).masterNodeId(master.localNode().id());
return ClusterState.builder(currentState).nodes(nodesBuilder).build();
ClusterState updatedState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
RoutingAllocation.Result routingResult = master.allocationService.reroute(ClusterState.builder(updatedState).build());
return ClusterState.builder(updatedState).routingResult(routingResult).build();
}
@Override

5
core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
View File

@ -932,7 +932,10 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
if (modified) {
stateBuilder.nodes(nodesBuilder);
}
return stateBuilder.build();
currentState = stateBuilder.build();
// eagerly run reroute to apply the node addition
RoutingAllocation.Result result = allocationService.reroute(currentState);
return ClusterState.builder(currentState).routingResult(result).build();
}
@Override

7
core/src/main/java/org/elasticsearch/discovery/zen/ping/multicast/MulticastZenPing.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.discovery.zen.ping.PingContextProvider;
import org.elasticsearch.discovery.zen.ping.ZenPing;
@ -408,9 +409,9 @@ public class MulticastZenPing extends AbstractLifecycleComponent<ZenPing> implem
xContentType = XContentFactory.xContentType(data);
if (xContentType != null) {
// an external ping
externalPingData = XContentFactory.xContent(xContentType)
.createParser(data)
.mapAndClose();
try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(data)) {
externalPingData = parser.map();
}
} else {
throw new IllegalStateException("failed multicast message, probably message from previous version");
}

53
core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
View File

@ -32,16 +32,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.MutableShardRouting;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.RoutingNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lease.Releasables;
@ -57,7 +55,6 @@ import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
/**
*
@ -70,8 +67,7 @@ public class GatewayAllocator extends AbstractComponent {
private final TransportNodesListGatewayStartedShards startedAction;
private final TransportNodesListShardStoreMetaData storeAction;
private ClusterService clusterService;
private AllocationService allocationService;
private RoutingService routingService;
private final ConcurrentMap<ShardId, AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards>> asyncFetchStarted = ConcurrentCollections.newConcurrentMap();
private final ConcurrentMap<ShardId, AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>> asyncFetchStore = ConcurrentCollections.newConcurrentMap();
@ -87,9 +83,8 @@ public class GatewayAllocator extends AbstractComponent {
logger.debug("using initial_shards [{}]", initialShards);
}
public void setReallocation(final ClusterService clusterService, final AllocationService allocationService) {
this.clusterService = clusterService;
this.allocationService = allocationService;
public void setReallocation(final ClusterService clusterService, final RoutingService routingService) {
this.routingService = routingService;
clusterService.add(new ClusterStateListener() {
@Override
public void clusterChanged(ClusterChangedEvent event) {
@ -153,9 +148,9 @@ public class GatewayAllocator extends AbstractComponent {
// First, handle primaries, they must find a place to be allocated on here
MetaData metaData = routingNodes.metaData();
Iterator<MutableShardRouting> unassignedIterator = routingNodes.unassigned().iterator();
Iterator<ShardRouting> unassignedIterator = routingNodes.unassigned().iterator();
while (unassignedIterator.hasNext()) {
MutableShardRouting shard = unassignedIterator.next();
ShardRouting shard = unassignedIterator.next();
if (!shard.primary()) {
continue;
@ -335,7 +330,7 @@ public class GatewayAllocator extends AbstractComponent {
// we found a match
changed = true;
// make sure we create one with the version from the recovered state
allocation.routingNodes().assign(new MutableShardRouting(shard, highestVersion), node.nodeId());
allocation.routingNodes().assign(new ShardRouting(shard, highestVersion), node.nodeId());
unassignedIterator.remove();
// found a node, so no throttling, no "no", and break out of the loop
@ -355,7 +350,7 @@ public class GatewayAllocator extends AbstractComponent {
// we found a match
changed = true;
// make sure we create one with the version from the recovered state
allocation.routingNodes().assign(new MutableShardRouting(shard, highestVersion), node.nodeId());
allocation.routingNodes().assign(new ShardRouting(shard, highestVersion), node.nodeId());
unassignedIterator.remove();
}
} else {
@ -375,7 +370,7 @@ public class GatewayAllocator extends AbstractComponent {
// Now, handle replicas, try to assign them to nodes that are similar to the one the primary was allocated on
unassignedIterator = routingNodes.unassigned().iterator();
while (unassignedIterator.hasNext()) {
MutableShardRouting shard = unassignedIterator.next();
ShardRouting shard = unassignedIterator.next();
if (shard.primary()) {
continue;
}
@ -453,7 +448,7 @@ public class GatewayAllocator extends AbstractComponent {
if (!shard.primary()) {
hasReplicaData |= storeFilesMetaData.iterator().hasNext();
MutableShardRouting primaryShard = routingNodes.activePrimary(shard);
ShardRouting primaryShard = routingNodes.activePrimary(shard);
if (primaryShard != null) {
assert primaryShard.active();
DiscoveryNode primaryNode = nodes.get(primaryShard.currentNodeId());
@ -535,8 +530,6 @@ public class GatewayAllocator extends AbstractComponent {
return changed;
}
private final AtomicBoolean rerouting = new AtomicBoolean();
class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
public InternalAsyncFetch(ESLogger logger, String type, ShardId shardId, List<? extends BaseNodesResponse<T>, T> action) {
@ -545,30 +538,8 @@ public class GatewayAllocator extends AbstractComponent {
@Override
protected void reroute(ShardId shardId, String reason) {
if (rerouting.compareAndSet(false, true) == false) {
logger.trace("{} already has pending reroute, ignoring {}", shardId, reason);
return;
}
clusterService.submitStateUpdateTask("async_shard_fetch", Priority.HIGH, new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
rerouting.set(false);
if (currentState.nodes().masterNode() == null) {
return currentState;
}
RoutingAllocation.Result routingResult = allocationService.reroute(currentState);
if (!routingResult.changed()) {
return currentState;
}
return ClusterState.builder(currentState).routingResult(routingResult).build();
}
@Override
public void onFailure(String source, Throwable t) {
rerouting.set(false);
logger.warn("failed to perform reroute post async fetch for {}", t, source);
}
});
logger.trace("{} scheduling reroute for {}", shardId, reason);
routingService.reroute("async_shard_fetch");
}
}
}

2
core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
View File

@ -284,7 +284,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state");
}
Set<String> indices = new HashSet<>();
for (MutableShardRouting routing : newRoutingNode) {
for (ShardRouting routing : newRoutingNode) {
indices.add(routing.index());
}
// we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously

19
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -36,12 +36,10 @@ import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.cache.filter.ShardFilterCache;
import org.elasticsearch.index.deletionpolicy.DeletionPolicyModule;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.gateway.IndexShardGatewayService;
import org.elasticsearch.index.shard.StoreRecoveryService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
@ -60,10 +58,8 @@ import org.elasticsearch.plugins.ShardsPluginsModule;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -306,12 +302,16 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
// if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false ||
(primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
final ShardFilterCache shardFilterCache = new ShardFilterCache(shardId, injector.getInstance(IndicesFilterCache.class));
ModulesBuilder modules = new ModulesBuilder();
modules.add(new ShardsPluginsModule(indexSettings, pluginsService));
modules.add(new IndexShardModule(shardId, primary, indexSettings, shardFilterCache));
modules.add(new IndexShardModule(shardId, primary, indexSettings));
modules.add(new StoreModule(injector.getInstance(IndexStore.class).shardDirectory(), lock,
new StoreCloseListener(shardId, canDeleteShardContent, shardFilterCache), path));
new StoreCloseListener(shardId, canDeleteShardContent, new Closeable() {
@Override
public void close() throws IOException {
injector.getInstance(IndicesFilterCache.class).onClose(shardId);
}
}), path));
modules.add(new DeletionPolicyModule(indexSettings));
try {
shardInjector = modules.createChildInjector(injector);
@ -387,8 +387,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
}
}
closeInjectorResource(sId, shardInjector,
IndexShardGatewayService.class,
PercolatorQueriesRegistry.class);
StoreRecoveryService.class);
// call this before we close the store, so we can release resources for it
indicesLifecycle.afterIndexShardClosed(sId, indexShard, indexSettings);

1
core/src/main/java/org/elasticsearch/index/cache/bitset/ShardBitsetFilterCache.java vendored
View File

@ -32,7 +32,6 @@ public class ShardBitsetFilterCache extends AbstractIndexShardComponent {
private final CounterMetric totalMetric = new CounterMetric();
@Inject
public ShardBitsetFilterCache(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
}

54
core/src/main/java/org/elasticsearch/index/cache/filter/ShardFilterCache.java vendored
View File

@ -1,54 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.cache.filter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
*/
public class ShardFilterCache implements Closeable {
final IndicesFilterCache cache;
final ShardId shardId;
public ShardFilterCache(ShardId shardId, IndicesFilterCache cache) {
this.cache = cache;
this.shardId = shardId;
}
public FilterCacheStats stats() {
return cache.getStats(shardId);
}
@Override
public void close() throws IOException {
cache.onClose(shardId);
}
}

1
core/src/main/java/org/elasticsearch/index/cache/query/ShardQueryCache.java vendored
View File

@ -39,7 +39,6 @@ public class ShardQueryCache extends AbstractIndexShardComponent implements Remo
final CounterMetric hitCount = new CounterMetric();
final CounterMetric missCount = new CounterMetric();
@Inject
public ShardQueryCache(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
}

11
core/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -45,7 +45,6 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy;
import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit;
import org.elasticsearch.index.fielddata.ShardFieldData;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.ParsedDocument;
@ -995,7 +994,6 @@ public abstract class Engine implements Closeable {
public static class Get {
private final boolean realtime;
private final Term uid;
private boolean loadSource = true;
private long version = Versions.MATCH_ANY;
private VersionType versionType = VersionType.INTERNAL;
@ -1012,15 +1010,6 @@ public abstract class Engine implements Closeable {
return uid;
}
public boolean loadSource() {
return this.loadSource;
}
public Get loadSource(boolean loadSource) {
this.loadSource = loadSource;
return this;
}
public long version() {
return version;
}

14
core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
View File

@ -53,6 +53,7 @@ import java.util.concurrent.TimeUnit;
public final class EngineConfig {
private final ShardId shardId;
private final TranslogRecoveryPerformer translogRecoveryPerformer;
private final Settings indexSettings;
private volatile ByteSizeValue indexingBufferSize;
private volatile ByteSizeValue versionMapSize;
private volatile String versionMapSizeSetting;
@ -64,7 +65,6 @@ public final class EngineConfig {
private final boolean optimizeAutoGenerateId;
private final ThreadPool threadPool;
private final ShardIndexingService indexingService;
private final IndexSettingsService indexSettingsService;
@Nullable
private final IndicesWarmer warmer;
private final Store store;
@ -141,14 +141,14 @@ public final class EngineConfig {
* Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
*/
public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService,
IndexSettingsService indexSettingsService, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer,
Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener,
TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache filterCache, QueryCachingPolicy filterCachingPolicy, TranslogConfig translogConfig) {
this.shardId = shardId;
this.indexSettings = indexSettings;
this.threadPool = threadPool;
this.indexingService = indexingService;
this.indexSettingsService = indexSettingsService;
this.warmer = warmer;
this.store = store;
this.deletionPolicy = deletionPolicy;
@ -158,7 +158,6 @@ public final class EngineConfig {
this.similarity = similarity;
this.codecService = codecService;
this.failedEngineListener = failedEngineListener;
Settings indexSettings = indexSettingsService.getSettings();
this.optimizeAutoGenerateId = indexSettings.getAsBoolean(EngineConfig.INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING, false);
this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
this.indexConcurrency = indexSettings.getAsInt(EngineConfig.INDEX_CONCURRENCY_SETTING, Math.max(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, (int) (EsExecutors.boundedNumberOfProcessors(indexSettings) * 0.65)));
@ -364,7 +363,7 @@ public final class EngineConfig {
* Returns the latest index settings directly from the index settings service.
*/
public Settings getIndexSettings() {
return indexSettingsService.getSettings();
return indexSettings;
}
/**
@ -429,9 +428,4 @@ public final class EngineConfig {
public TranslogConfig getTranslogConfig() {
return translogConfig;
}
IndexSettingsService getIndexSettingsService() { // for testing
return indexSettingsService;
}
}

3
core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View File

@ -305,9 +305,6 @@ public class InternalEngine extends Engine {
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), versionValue.version(), get.version());
}
if (!get.loadSource()) {
return new GetResult(true, versionValue.version(), null);
}
Translog.Operation op = translog.read(versionValue.translogLocation());
if (op != null) {
return new GetResult(true, versionValue.version(), op.getSource());

8
core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
View File

@ -36,18 +36,12 @@ import java.util.concurrent.ConcurrentMap;
/**
*/
public class ShardFieldData extends AbstractIndexShardComponent implements IndexFieldDataCache.Listener {
public class ShardFieldData implements IndexFieldDataCache.Listener {
final CounterMetric evictionsMetric = new CounterMetric();
final CounterMetric totalMetric = new CounterMetric();
final ConcurrentMap<String, CounterMetric> perFieldTotals = ConcurrentCollections.newConcurrentMap();
@Inject
public ShardFieldData(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
}
public FieldDataStats stats(String... fields) {
ObjectLongHashMap<String> fieldTotals = null;
if (fields != null && fields.length > 0) {

32
core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldDataModule.java
View File

@ -1,32 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.inject.AbstractModule;
/**
*/
public class ShardFieldDataModule extends AbstractModule {
@Override
protected void configure() {
bind(ShardFieldData.class).asEagerSingleton();
}
}

1
core/src/main/java/org/elasticsearch/index/fieldvisitor/AllFieldsVisitor.java
View File

@ -27,6 +27,7 @@ import java.io.IOException;
public class AllFieldsVisitor extends FieldsVisitor {
public AllFieldsVisitor() {
super(true);
}
@Override

11
core/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
import java.util.Set;
@ -32,21 +30,16 @@ import java.util.Set;
*/
public class CustomFieldsVisitor extends FieldsVisitor {
private final boolean loadSource;
private final Set<String> fields;
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
this.loadSource = loadSource;
super(loadSource);
this.fields = fields;
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (loadSource && SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
if (super.needsField(fieldInfo) == Status.YES) {
return Status.YES;
}

57
core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fieldvisitor;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.StoredFieldVisitor;
@ -27,30 +28,63 @@ import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static com.google.common.collect.Maps.newHashMap;
/**
* Base {@link StoredFieldsVisitor} that retrieves all non-redundant metadata.
*/
public abstract class FieldsVisitor extends StoredFieldVisitor {
public class FieldsVisitor extends StoredFieldVisitor {
private static final Set<String> BASE_REQUIRED_FIELDS = ImmutableSet.of(
UidFieldMapper.NAME,
TimestampFieldMapper.NAME,
TTLFieldMapper.NAME,
RoutingFieldMapper.NAME,
ParentFieldMapper.NAME
);
private final boolean loadSource;
private final Set<String> requiredFields;
protected BytesReference source;
protected Uid uid;
protected Map<String, List<Object>> fieldsValues;
public FieldsVisitor(boolean loadSource) {
this.loadSource = loadSource;
requiredFields = new HashSet<>();
reset();
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (requiredFields.remove(fieldInfo.name)) {
return Status.YES;
}
// All these fields are single-valued so we can stop when the set is
// empty
return requiredFields.isEmpty()
? Status.STOP
: Status.NO;
}
public void postProcess(MapperService mapperService) {
if (uid != null) {
DocumentMapper documentMapper = mapperService.documentMapper(uid.type());
@ -144,6 +178,18 @@ public abstract class FieldsVisitor extends StoredFieldVisitor {
return uid;
}
public String routing() {
if (fieldsValues == null) {
return null;
}
List<Object> values = fieldsValues.get(RoutingFieldMapper.NAME);
if (values == null || values.isEmpty()) {
return null;
}
assert values.size() == 1;
return values.get(0).toString();
}
public Map<String, List<Object>> fields() {
return fieldsValues != null
? fieldsValues
@ -154,6 +200,11 @@ public abstract class FieldsVisitor extends StoredFieldVisitor {
if (fieldsValues != null) fieldsValues.clear();
source = null;
uid = null;
requiredFields.addAll(BASE_REQUIRED_FIELDS);
if (loadSource) {
requiredFields.add(SourceFieldMapper.NAME);
}
}
void addValue(String name, Object value) {

37
core/src/main/java/org/elasticsearch/index/fieldvisitor/JustSourceFieldsVisitor.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import java.io.IOException;
/**
*/
public class JustSourceFieldsVisitor extends FieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return source != null ? Status.STOP : Status.NO;
}
}

4
core/src/main/java/org/elasticsearch/index/fieldvisitor/JustUidFieldsVisitor.java
View File

@ -27,6 +27,10 @@ import java.io.IOException;
*/
public class JustUidFieldsVisitor extends FieldsVisitor {
public JustUidFieldsVisitor() {
super(false);
}
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {

1
core/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java
View File

@ -35,6 +35,7 @@ public class SingleFieldsVisitor extends FieldsVisitor {
private String field;
public SingleFieldsVisitor(String field) {
super(false);
this.field = field;
}

57
core/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndRoutingFieldsVisitor.java
View File

@ -1,57 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
*/
public class UidAndRoutingFieldsVisitor extends FieldsVisitor {
private String routing;
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && routing != null ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
if (RoutingFieldMapper.NAME.equals(fieldInfo.name)) {
routing = new String(bytes, StandardCharsets.UTF_8);;
} else {
super.stringField(fieldInfo, bytes);
}
}
public String routing() {
return routing;
}
}

41
core/src/main/java/org/elasticsearch/index/fieldvisitor/UidAndSourceFieldsVisitor.java
View File

@ -1,41 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fieldvisitor;
import org.apache.lucene.index.FieldInfo;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import java.io.IOException;
/**
*/
public class UidAndSourceFieldsVisitor extends FieldsVisitor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
} else if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
return Status.YES;
}
return uid != null && source != null ? Status.STOP : Status.NO;
}
}

39
core/src/main/java/org/elasticsearch/index/gateway/IgnoreGatewayRecoveryException.java
View File

@ -1,39 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.elasticsearch.index.shard.IndexShardException;
import org.elasticsearch.index.shard.ShardId;
/**
* An exception marking that this recovery attempt should be ignored (since probably, we already recovered).
*
*
*/
public class IgnoreGatewayRecoveryException extends IndexShardException {
public IgnoreGatewayRecoveryException(ShardId shardId, String msg) {
super(shardId, msg);
}
public IgnoreGatewayRecoveryException(ShardId shardId, String msg, Throwable cause) {
super(shardId, msg, cause);
}
}

192
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGateway.java
View File

@ -1,192 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.Directory;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.CancellableThreads;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.mapper.Mapping;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.recovery.RecoveryState;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
/**
*
*/
public class IndexShardGateway extends AbstractIndexShardComponent implements Closeable {
private final MappingUpdatedAction mappingUpdatedAction;
private final IndexService indexService;
private final IndexShard indexShard;
private final TimeValue waitForMappingUpdatePostRecovery;
private final CancellableThreads cancellableThreads = new CancellableThreads();
@Inject
public IndexShardGateway(ShardId shardId, @IndexSettings Settings indexSettings, MappingUpdatedAction mappingUpdatedAction,
IndexService indexService, IndexShard indexShard) {
super(shardId, indexSettings);
this.mappingUpdatedAction = mappingUpdatedAction;
this.indexService = indexService;
this.indexShard = indexShard;
this.waitForMappingUpdatePostRecovery = indexSettings.getAsTime("index.gateway.wait_for_mapping_update_post_recovery", TimeValue.timeValueSeconds(15));
}
/**
* Recovers the state of the shard from the gateway.
*/
public void recover(boolean indexShouldExists, RecoveryState recoveryState) throws IndexShardGatewayRecoveryException {
indexShard.prepareForIndexRecovery();
long version = -1;
final Map<String, Mapping> typesToUpdate;
SegmentInfos si = null;
final Store store = indexShard.store();
store.incRef();
try {
try {
store.failIfCorrupted();
try {
si = store.readLastCommittedSegmentsInfo();
} catch (Throwable e) {
String files = "_unknown_";
try {
files = Arrays.toString(store.directory().listAll());
} catch (Throwable e1) {
files += " (failure=" + ExceptionsHelper.detailedMessage(e1) + ")";
}
if (indexShouldExists) {
throw new IndexShardGatewayRecoveryException(shardId(), "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files, e);
}
}
if (si != null) {
if (indexShouldExists) {
version = si.getVersion();
} else {
// it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
// its a "new index create" API, we have to do something, so better to clean it than use same data
logger.trace("cleaning existing shard, shouldn't exists");
IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
writer.close();
recoveryState.getTranslog().totalOperations(0);
}
}
} catch (Throwable e) {
throw new IndexShardGatewayRecoveryException(shardId(), "failed to fetch index version after copying it over", e);
}
recoveryState.getIndex().updateVersion(version);
// since we recover from local, just fill the files and size
try {
final RecoveryState.Index index = recoveryState.getIndex();
if (si != null) {
final Directory directory = store.directory();
for (String name : Lucene.files(si)) {
long length = directory.fileLength(name);
index.addFileDetail(name, length, true);
}
}
} catch (IOException e) {
logger.debug("failed to list file details", e);
}
if (indexShouldExists == false) {
recoveryState.getTranslog().totalOperations(0);
recoveryState.getTranslog().totalOperationsOnStart(0);
}
typesToUpdate = indexShard.performTranslogRecovery();
indexShard.finalizeRecovery();
for (Map.Entry<String, Mapping> entry : typesToUpdate.entrySet()) {
validateMappingUpdate(entry.getKey(), entry.getValue());
}
indexShard.postRecovery("post recovery from gateway");
} catch (EngineException e) {
throw new IndexShardGatewayRecoveryException(shardId, "failed to recovery from gateway", e);
} finally {
store.decRef();
}
}
private void validateMappingUpdate(final String type, Mapping update) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> error = new AtomicReference<>();
mappingUpdatedAction.updateMappingOnMaster(indexService.index().name(), type, update, waitForMappingUpdatePostRecovery, new MappingUpdatedAction.MappingUpdateListener() {
@Override
public void onMappingUpdate() {
latch.countDown();
}
@Override
public void onFailure(Throwable t) {
latch.countDown();
error.set(t);
}
});
cancellableThreads.execute(new CancellableThreads.Interruptable() {
@Override
public void run() throws InterruptedException {
try {
if (latch.await(waitForMappingUpdatePostRecovery.millis(), TimeUnit.MILLISECONDS) == false) {
logger.debug("waited for mapping update on master for [{}], yet timed out", type);
} else {
if (error.get() != null) {
throw new IndexShardGatewayRecoveryException(shardId, "Failed to propagate mappings on master post recovery", error.get());
}
}
} catch (InterruptedException e) {
logger.debug("interrupted while waiting for mapping update");
throw e;
}
}
});
}
@Override
public void close() {
cancellableThreads.cancel("closed");
}
@Override
public String toString() {
return "shard_gateway";
}
}

37
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayException.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.elasticsearch.index.shard.IndexShardException;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
public class IndexShardGatewayException extends IndexShardException {
public IndexShardGatewayException(ShardId shardId, String msg) {
super(shardId, msg);
}
public IndexShardGatewayException(ShardId shardId, String msg, Throwable cause) {
super(shardId, msg, cause);
}
}

37
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayRecoveryException.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
public class IndexShardGatewayRecoveryException extends IndexShardGatewayException {
public IndexShardGatewayRecoveryException(ShardId shardId, String msg) {
super(shardId, msg);
}
public IndexShardGatewayRecoveryException(ShardId shardId, String msg, Throwable cause) {
super(shardId, msg, cause);
}
}

174
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewayService.java
View File

@ -1,174 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.*;
import org.elasticsearch.index.snapshots.IndexShardSnapshotAndRestoreService;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
/**
*
*/
public class IndexShardGatewayService extends AbstractIndexShardComponent implements Closeable {
private final ThreadPool threadPool;
private final ClusterService clusterService;
private final IndexShard indexShard;
private final IndexShardGateway shardGateway;
private final IndexShardSnapshotAndRestoreService snapshotService;
@Inject
public IndexShardGatewayService(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool,
IndexShard indexShard, IndexShardGateway shardGateway, IndexShardSnapshotAndRestoreService snapshotService, ClusterService clusterService) {
super(shardId, indexSettings);
this.threadPool = threadPool;
this.indexShard = indexShard;
this.shardGateway = shardGateway;
this.snapshotService = snapshotService;
this.clusterService = clusterService;
}
/**
* Should be called when the shard routing state has changed (note, after the state has been set on the shard).
*/
public void routingStateChanged() {
}
public static interface RecoveryListener {
void onRecoveryDone();
void onIgnoreRecovery(String reason);
void onRecoveryFailed(IndexShardGatewayRecoveryException e);
}
/**
* Recovers the state of the shard from the gateway.
*/
public void recover(final boolean indexShouldExists, final RecoveryListener listener) throws IndexShardGatewayRecoveryException, IgnoreGatewayRecoveryException {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if (!indexShard.routingEntry().primary()) {
listener.onRecoveryFailed(new IndexShardGatewayRecoveryException(shardId, "Trying to recover when the shard is in backup state", null));
return;
}
try {
if (indexShard.routingEntry().restoreSource() != null) {
indexShard.recovering("from snapshot", RecoveryState.Type.SNAPSHOT, indexShard.routingEntry().restoreSource());
} else {
indexShard.recovering("from gateway", RecoveryState.Type.GATEWAY, clusterService.localNode());
}
} catch (IllegalIndexShardStateException e) {
// that's fine, since we might be called concurrently, just ignore this, we are already recovering
listener.onIgnoreRecovery("already in recovering process, " + e.getMessage());
return;
}
threadPool.generic().execute(new Runnable() {
@Override
public void run() {
try {
final RecoveryState recoveryState = indexShard.recoveryState();
if (indexShard.routingEntry().restoreSource() != null) {
logger.debug("restoring from {} ...", indexShard.routingEntry().restoreSource());
snapshotService.restore(recoveryState);
} else {
logger.debug("starting recovery from {} ...", shardGateway);
shardGateway.recover(indexShouldExists, recoveryState);
}
// Check that the gateway didn't leave the shard in init or recovering stage. it is up to the gateway
// to call post recovery.
IndexShardState shardState = indexShard.state();
assert shardState != IndexShardState.CREATED && shardState != IndexShardState.RECOVERING : "recovery process of " + shardId + " didn't get to post_recovery. shardState [" + shardState + "]";
if (logger.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append("recovery completed from ").append(shardGateway).append(", took [").append(timeValueMillis(recoveryState.getTimer().time())).append("]\n");
RecoveryState.Index index = recoveryState.getIndex();
sb.append(" index : files [").append(index.totalFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.totalBytes())).append("], took[")
.append(TimeValue.timeValueMillis(index.time())).append("]\n");
sb.append(" : recovered_files [").append(index.recoveredFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.recoveredBytes())).append("]\n");
sb.append(" : reusing_files [").append(index.reusedFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.reusedBytes())).append("]\n");
sb.append(" verify_index : took [").append(TimeValue.timeValueMillis(recoveryState.getVerifyIndex().time())).append("], check_index [")
.append(timeValueMillis(recoveryState.getVerifyIndex().checkIndexTime())).append("]\n");
sb.append(" translog : number_of_operations [").append(recoveryState.getTranslog().recoveredOperations())
.append("], took [").append(TimeValue.timeValueMillis(recoveryState.getTranslog().time())).append("]");
logger.trace(sb.toString());
} else if (logger.isDebugEnabled()) {
logger.debug("recovery completed from [{}], took [{}]", shardGateway, timeValueMillis(recoveryState.getTimer().time()));
}
listener.onRecoveryDone();
} catch (IndexShardGatewayRecoveryException e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if ((e.getCause() instanceof IndexShardClosedException) || (e.getCause() instanceof IndexShardNotStartedException)) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(e);
} catch (IndexShardClosedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (IndexShardNotStartedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (Exception e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(new IndexShardGatewayRecoveryException(shardId, "failed recovery", e));
}
}
});
}
@Override
public synchronized void close() throws IOException {
shardGateway.close();
}
}

32
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewaySnapshotNotAllowedException.java
View File

@ -1,32 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.gateway;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
public class IndexShardGatewaySnapshotNotAllowedException extends IndexShardGatewayException {
public IndexShardGatewaySnapshotNotAllowedException(ShardId shardId, String msg) {
super(shardId, msg);
}
}

130
core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -27,37 +27,33 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.lookup.LeafSearchLookup;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static com.google.common.collect.Maps.newHashMapWithExpectedSize;
@ -154,13 +150,11 @@ public final class ShardGetService extends AbstractIndexShardComponent {
private GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext, boolean ignoreErrorsOnGeneratedFields) {
fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, gFields);
boolean loadSource = (gFields != null && gFields.length > 0) || fetchSourceContext.fetchSource();
Engine.GetResult get = null;
if (type == null || type.equals("_all")) {
for (String typeX : mapperService.types()) {
get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(typeX, id)))
.loadSource(loadSource).version(version).versionType(versionType));
.version(version).versionType(versionType));
if (get.exists()) {
type = typeX;
break;
@ -177,7 +171,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
}
} else {
get = indexShard.get(new Engine.Get(realtime, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(type, id)))
.loadSource(loadSource).version(version).versionType(versionType));
.version(version).versionType(versionType));
if (!get.exists()) {
get.release();
return new GetResult(shardId.index().name(), type, id, -1, false, null, null);
@ -201,58 +195,72 @@ public final class ShardGetService extends AbstractIndexShardComponent {
SearchLookup searchLookup = null;
// we can only load scripts that can run against the source
if (gFields != null && gFields.length > 0) {
for (String field : gFields) {
if (SourceFieldMapper.NAME.equals(field)) {
// dealt with when normalizing fetchSourceContext.
continue;
Set<String> neededFields = new HashSet<>();
// add meta fields
neededFields.add(RoutingFieldMapper.NAME);
if (docMapper.parentFieldMapper().active()) {
neededFields.add(ParentFieldMapper.NAME);
}
if (docMapper.timestampFieldMapper().enabled()) {
neededFields.add(TimestampFieldMapper.NAME);
}
if (docMapper.TTLFieldMapper().enabled()) {
neededFields.add(TTLFieldMapper.NAME);
}
// add requested fields
if (gFields != null) {
neededFields.addAll(Arrays.asList(gFields));
}
for (String field : neededFields) {
if (SourceFieldMapper.NAME.equals(field)) {
// dealt with when normalizing fetchSourceContext.
continue;
}
Object value = null;
if (field.equals(RoutingFieldMapper.NAME)) {
value = source.routing;
} else if (field.equals(ParentFieldMapper.NAME) && docMapper.parentFieldMapper().active()) {
value = source.parent;
} else if (field.equals(TimestampFieldMapper.NAME) && docMapper.timestampFieldMapper().enabled()) {
value = source.timestamp;
} else if (field.equals(TTLFieldMapper.NAME) && docMapper.TTLFieldMapper().enabled()) {
// Call value for search with timestamp + ttl here to display the live remaining ttl value and be consistent with the search result display
if (source.ttl > 0) {
value = docMapper.TTLFieldMapper().valueForSearch(source.timestamp + source.ttl);
}
Object value = null;
if (field.equals(RoutingFieldMapper.NAME) && docMapper.routingFieldMapper().fieldType().stored()) {
value = source.routing;
} else if (field.equals(ParentFieldMapper.NAME) && docMapper.parentFieldMapper().active() && docMapper.parentFieldMapper().fieldType().stored()) {
value = source.parent;
} else if (field.equals(TimestampFieldMapper.NAME) && docMapper.timestampFieldMapper().fieldType().stored()) {
value = source.timestamp;
} else if (field.equals(TTLFieldMapper.NAME) && docMapper.TTLFieldMapper().fieldType().stored()) {
// Call value for search with timestamp + ttl here to display the live remaining ttl value and be consistent with the search result display
if (source.ttl > 0) {
value = docMapper.TTLFieldMapper().valueForSearch(source.timestamp + source.ttl);
} else if (field.equals(SizeFieldMapper.NAME) && docMapper.rootMapper(SizeFieldMapper.class).fieldType().stored()) {
value = source.source.length();
} else {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, null, new String[]{type});
searchLookup.source().setSource(source.source);
}
FieldMapper fieldMapper = docMapper.mappers().smartNameFieldMapper(field);
if (fieldMapper == null) {
if (docMapper.objectMappers().get(field) != null) {
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
}
} else if (field.equals(SizeFieldMapper.NAME) && docMapper.rootMapper(SizeFieldMapper.class).fieldType().stored()) {
value = source.source.length();
} else if (shouldGetFromSource(ignoreErrorsOnGeneratedFields, docMapper, fieldMapper)) {
List<Object> values = searchLookup.source().extractRawValues(field);
if (!values.isEmpty()) {
for (int i = 0; i < values.size(); i++) {
values.set(i, fieldMapper.fieldType().valueForSearch(values.get(i)));
}
value = values;
}
}
}
if (value != null) {
if (fields == null) {
fields = newHashMapWithExpectedSize(2);
}
if (value instanceof List) {
fields.put(field, new GetField(field, (List) value));
} else {
if (searchLookup == null) {
searchLookup = new SearchLookup(mapperService, null, new String[]{type});
searchLookup.source().setSource(source.source);
}
FieldMapper fieldMapper = docMapper.mappers().smartNameFieldMapper(field);
if (fieldMapper == null) {
if (docMapper.objectMappers().get(field) != null) {
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
}
} else if (shouldGetFromSource(ignoreErrorsOnGeneratedFields, docMapper, fieldMapper)) {
List<Object> values = searchLookup.source().extractRawValues(field);
if (!values.isEmpty()) {
for (int i = 0; i < values.size(); i++) {
values.set(i, fieldMapper.fieldType().valueForSearch(values.get(i)));
}
value = values;
}
}
}
if (value != null) {
if (fields == null) {
fields = newHashMapWithExpectedSize(2);
}
if (value instanceof List) {
fields.put(field, new GetField(field, (List) value));
} else {
fields.put(field, new GetField(field, ImmutableList.of(value)));
}
fields.put(field, new GetField(field, ImmutableList.of(value)));
}
}
}
@ -408,7 +416,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
private static FieldsVisitor buildFieldsVisitors(String[] fields, FetchSourceContext fetchSourceContext) {
if (fields == null || fields.length == 0) {
return fetchSourceContext.fetchSource() ? new JustSourceFieldsVisitor() : null;
return fetchSourceContext.fetchSource() ? new FieldsVisitor(true) : null;
}
return new CustomFieldsVisitor(Sets.newHashSet(fields), fetchSourceContext.fetchSource());

106
core/src/main/java/org/elasticsearch/index/indexing/slowlog/ShardSlowLogIndexingService.java → core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java
View File

@ -17,9 +17,8 @@
* under the License.
*/
package org.elasticsearch.index.indexing.slowlog;
package org.elasticsearch.index.indexing;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
@ -27,10 +26,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.util.Locale;
@ -38,7 +34,7 @@ import java.util.concurrent.TimeUnit;
/**
*/
public class ShardSlowLogIndexingService extends AbstractIndexShardComponent {
public final class IndexingSlowLog {
private boolean reformat;
@ -52,53 +48,16 @@ public class ShardSlowLogIndexingService extends AbstractIndexShardComponent {
private final ESLogger indexLogger;
private final ESLogger deleteLogger;
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN = "index.indexing.slowlog.threshold.index.warn";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO = "index.indexing.slowlog.threshold.index.info";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG = "index.indexing.slowlog.threshold.index.debug";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE = "index.indexing.slowlog.threshold.index.trace";
public static final String INDEX_INDEXING_SLOWLOG_REFORMAT = "index.indexing.slowlog.reformat";
public static final String INDEX_INDEXING_SLOWLOG_LEVEL = "index.indexing.slowlog.level";
class ApplySettings implements IndexSettingsService.Listener {
@Override
public synchronized void onRefreshSettings(Settings settings) {
long indexWarnThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, TimeValue.timeValueNanos(ShardSlowLogIndexingService.this.indexWarnThreshold)).nanos();
if (indexWarnThreshold != ShardSlowLogIndexingService.this.indexWarnThreshold) {
ShardSlowLogIndexingService.this.indexWarnThreshold = indexWarnThreshold;
}
long indexInfoThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, TimeValue.timeValueNanos(ShardSlowLogIndexingService.this.indexInfoThreshold)).nanos();
if (indexInfoThreshold != ShardSlowLogIndexingService.this.indexInfoThreshold) {
ShardSlowLogIndexingService.this.indexInfoThreshold = indexInfoThreshold;
}
long indexDebugThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, TimeValue.timeValueNanos(ShardSlowLogIndexingService.this.indexDebugThreshold)).nanos();
if (indexDebugThreshold != ShardSlowLogIndexingService.this.indexDebugThreshold) {
ShardSlowLogIndexingService.this.indexDebugThreshold = indexDebugThreshold;
}
long indexTraceThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE, TimeValue.timeValueNanos(ShardSlowLogIndexingService.this.indexTraceThreshold)).nanos();
if (indexTraceThreshold != ShardSlowLogIndexingService.this.indexTraceThreshold) {
ShardSlowLogIndexingService.this.indexTraceThreshold = indexTraceThreshold;
}
String level = settings.get(INDEX_INDEXING_SLOWLOG_LEVEL, ShardSlowLogIndexingService.this.level);
if (!level.equals(ShardSlowLogIndexingService.this.level)) {
ShardSlowLogIndexingService.this.indexLogger.setLevel(level.toUpperCase(Locale.ROOT));
ShardSlowLogIndexingService.this.deleteLogger.setLevel(level.toUpperCase(Locale.ROOT));
ShardSlowLogIndexingService.this.level = level;
}
boolean reformat = settings.getAsBoolean(INDEX_INDEXING_SLOWLOG_REFORMAT, ShardSlowLogIndexingService.this.reformat);
if (reformat != ShardSlowLogIndexingService.this.reformat) {
ShardSlowLogIndexingService.this.reformat = reformat;
}
}
}
@Inject
public ShardSlowLogIndexingService(ShardId shardId, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) {
super(shardId, indexSettings);
private static final String INDEX_INDEXING_SLOWLOG_PREFIX = "index.indexing.slowlog";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN = INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.warn";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO = INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.info";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG = INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.debug";
public static final String INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE = INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.trace";
public static final String INDEX_INDEXING_SLOWLOG_REFORMAT = INDEX_INDEXING_SLOWLOG_PREFIX +".reformat";
public static final String INDEX_INDEXING_SLOWLOG_LEVEL = INDEX_INDEXING_SLOWLOG_PREFIX +".level";
IndexingSlowLog(Settings indexSettings) {
this.reformat = indexSettings.getAsBoolean(INDEX_INDEXING_SLOWLOG_REFORMAT, true);
this.indexWarnThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, TimeValue.timeValueNanos(-1)).nanos();
this.indexInfoThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, TimeValue.timeValueNanos(-1)).nanos();
this.indexDebugThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, TimeValue.timeValueNanos(-1)).nanos();
@ -106,20 +65,49 @@ public class ShardSlowLogIndexingService extends AbstractIndexShardComponent {
this.level = indexSettings.get(INDEX_INDEXING_SLOWLOG_LEVEL, "TRACE").toUpperCase(Locale.ROOT);
this.indexLogger = Loggers.getLogger(logger, ".index");
this.deleteLogger = Loggers.getLogger(logger, ".delete");
this.indexLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX +".index");
this.deleteLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX +".delete");
indexLogger.setLevel(level);
deleteLogger.setLevel(level);
indexSettingsService.addListener(new ApplySettings());
}
public void postIndex(Engine.Index index, long tookInNanos) {
synchronized void onRefreshSettings(Settings settings) {
long indexWarnThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, TimeValue.timeValueNanos(this.indexWarnThreshold)).nanos();
if (indexWarnThreshold != this.indexWarnThreshold) {
this.indexWarnThreshold = indexWarnThreshold;
}
long indexInfoThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, TimeValue.timeValueNanos(this.indexInfoThreshold)).nanos();
if (indexInfoThreshold != this.indexInfoThreshold) {
this.indexInfoThreshold = indexInfoThreshold;
}
long indexDebugThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, TimeValue.timeValueNanos(this.indexDebugThreshold)).nanos();
if (indexDebugThreshold != this.indexDebugThreshold) {
this.indexDebugThreshold = indexDebugThreshold;
}
long indexTraceThreshold = settings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE, TimeValue.timeValueNanos(this.indexTraceThreshold)).nanos();
if (indexTraceThreshold != this.indexTraceThreshold) {
this.indexTraceThreshold = indexTraceThreshold;
}
String level = settings.get(INDEX_INDEXING_SLOWLOG_LEVEL, this.level);
if (!level.equals(this.level)) {
this.indexLogger.setLevel(level.toUpperCase(Locale.ROOT));
this.deleteLogger.setLevel(level.toUpperCase(Locale.ROOT));
this.level = level;
}
boolean reformat = settings.getAsBoolean(INDEX_INDEXING_SLOWLOG_REFORMAT, this.reformat);
if (reformat != this.reformat) {
this.reformat = reformat;
}
}
void postIndex(Engine.Index index, long tookInNanos) {
postIndexing(index.parsedDoc(), tookInNanos);
}
public void postCreate(Engine.Create create, long tookInNanos) {
void postCreate(Engine.Create create, long tookInNanos) {
postIndexing(create.parsedDoc(), tookInNanos);
}
@ -135,12 +123,12 @@ public class ShardSlowLogIndexingService extends AbstractIndexShardComponent {
}
}
public static class SlowLogParsedDocumentPrinter {
final static class SlowLogParsedDocumentPrinter {
private final ParsedDocument doc;
private final long tookInNanos;
private final boolean reformat;
public SlowLogParsedDocumentPrinter(ParsedDocument doc, long tookInNanos, boolean reformat) {
SlowLogParsedDocumentPrinter(ParsedDocument doc, long tookInNanos, boolean reformat) {
this.doc = doc;
this.tookInNanos = tookInNanos;
this.reformat = reformat;

14
core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java
View File

@ -21,15 +21,12 @@ package org.elasticsearch.index.indexing;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
@ -42,7 +39,7 @@ import java.util.concurrent.TimeUnit;
*/
public class ShardIndexingService extends AbstractIndexShardComponent {
private final ShardSlowLogIndexingService slowLog;
private final IndexingSlowLog slowLog;
private final StatsHolder totalStats = new StatsHolder();
@ -50,10 +47,9 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
private volatile Map<String, StatsHolder> typesStats = ImmutableMap.of();
@Inject
public ShardIndexingService(ShardId shardId, @IndexSettings Settings indexSettings, ShardSlowLogIndexingService slowLog) {
public ShardIndexingService(ShardId shardId, Settings indexSettings) {
super(shardId, indexSettings);
this.slowLog = slowLog;
this.slowLog = new IndexingSlowLog(indexSettings);
}
/**
@ -252,6 +248,10 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
return stats;
}
public void onRefreshSettings(Settings settings) {
slowLog.onRefreshSettings(settings);
}
static class StatsHolder {
public final MeanMetric indexMetric = new MeanMetric();
public final MeanMetric deleteMetric = new MeanMetric();

5
core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
@ -304,8 +305,8 @@ public class DocumentMapperParser extends AbstractIndexComponent {
private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
Map<String, Object> root;
try {
root = XContentFactory.xContent(source).createParser(source).mapOrderedAndClose();
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
root = parser.mapOrdered();
} catch (Exception e) {
throw new MapperParsingException("failed to parse mapping definition", e);
}

5
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -751,7 +751,10 @@ class DocumentParser implements Closeable {
}
private static XContentParser transform(Mapping mapping, XContentParser parser) throws IOException {
Map<String, Object> transformed = transformSourceAsMap(mapping, parser.mapOrderedAndClose());
Map<String, Object> transformed;
try (XContentParser _ = parser) {
transformed = transformSourceAsMap(mapping, parser.mapOrdered());
}
XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()).value(transformed);
return parser.contentType().xContent().createParser(builder.bytes());
}

3
core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
View File

@ -26,14 +26,12 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CloseableThreadLocal;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.indexing.IndexingOperationListener;
@ -94,7 +92,6 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple
}
};
@Inject
public PercolatorQueriesRegistry(ShardId shardId, @IndexSettings Settings indexSettings, IndexQueryParserService queryParserService,
ShardIndexingService indexingService, IndicesLifecycle indicesLifecycle, MapperService mapperService,
IndexFieldDataService indexFieldDataService, ShardPercolateService shardPercolateService) {

6
core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.percolator;
import com.google.common.collect.Maps;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Query;
@ -29,8 +30,7 @@ import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Uid;
@ -44,7 +44,7 @@ import java.util.Map;
final class QueriesLoaderCollector extends SimpleCollector {
private final Map<BytesRef, Query> queries = Maps.newHashMap();
private final JustSourceFieldsVisitor fieldsVisitor = new JustSourceFieldsVisitor();
private final FieldsVisitor fieldsVisitor = new FieldsVisitor(true);
private final PercolatorQueriesRegistry percolator;
private final IndexFieldData<?> uidFieldData;
private final ESLogger logger;

2
core/src/main/java/org/elasticsearch/index/query/BoostableQueryBuilder.java
View File

@ -28,6 +28,6 @@ public interface BoostableQueryBuilder<B extends BoostableQueryBuilder<B>> {
* Sets the boost for this query. Documents matching this query will (in addition to the normal
* weightings) have their score multiplied by the boost provided.
*/
public B boost(float boost);
B boost(float boost);
}

14
core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
View File

@ -29,7 +29,7 @@ import java.io.IOException;
/**
* {@link QueryBuilder} that builds a GeoShape Filter
*/
public class GeoShapeQueryBuilder extends QueryBuilder {
public class GeoShapeQueryBuilder extends QueryBuilder implements BoostableQueryBuilder<GeoShapeQueryBuilder> {
private final String name;
@ -46,6 +46,8 @@ public class GeoShapeQueryBuilder extends QueryBuilder {
private String indexedShapePath;
private ShapeRelation relation = null;
private float boost = -1;
/**
* Creates a new GeoShapeQueryBuilder whose Filter will be against the
@ -146,6 +148,12 @@ public class GeoShapeQueryBuilder extends QueryBuilder {
return this;
}
@Override
public GeoShapeQueryBuilder boost(float boost) {
this.boost = boost;
return this;
}
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(GeoShapeQueryParser.NAME);
@ -177,6 +185,10 @@ public class GeoShapeQueryBuilder extends QueryBuilder {
builder.endObject();
if (boost != -1) {
builder.field("boost", boost);
}
if (name != null) {
builder.field("_name", queryName);
}

4
core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View File

@ -204,6 +204,10 @@ public class QueryParseContext {
public void addInnerHits(String name, InnerHitsContext.BaseInnerHits context) {
SearchContext sc = SearchContext.current();
if (sc == null) {
throw new QueryParsingException(this, "inner_hits unsupported");
}
InnerHitsContext innerHitsContext;
if (sc.innerHits() == null) {
innerHitsContext = new InnerHitsContext(new HashMap<String, InnerHitsContext.BaseInnerHits>());

14
core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
View File

@ -26,7 +26,7 @@ import java.io.IOException;
/**
* A filer for a field based on several terms matching on any of them.
*/
public class TermsQueryBuilder extends QueryBuilder {
public class TermsQueryBuilder extends QueryBuilder implements BoostableQueryBuilder<TermsQueryBuilder> {
private final String name;
@ -36,6 +36,8 @@ public class TermsQueryBuilder extends QueryBuilder {
private String execution;
private float boost = -1;
/**
* A filer for a field based on several terms matching on any of them.
*
@ -131,6 +133,12 @@ public class TermsQueryBuilder extends QueryBuilder {
return this;
}
@Override
public TermsQueryBuilder boost(float boost) {
this.boost = boost;
return this;
}
@Override
public void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(TermsQueryParser.NAME);
@ -140,6 +148,10 @@ public class TermsQueryBuilder extends QueryBuilder {
builder.field("execution", execution);
}
if (boost != -1) {
builder.field("boost", boost);
}
if (queryName != null) {
builder.field("_name", queryName);
}

145
core/src/main/java/org/elasticsearch/index/search/slowlog/ShardSlowLogSearchService.java → core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java
View File

@ -17,19 +17,14 @@
* under the License.
*/
package org.elasticsearch.index.search.slowlog;
package org.elasticsearch.index.search.stats;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
@ -38,7 +33,7 @@ import java.util.concurrent.TimeUnit;
/**
*/
public class ShardSlowLogSearchService extends AbstractIndexShardComponent {
public final class SearchSlowLog{
private boolean reformat;
@ -57,71 +52,19 @@ public class ShardSlowLogSearchService extends AbstractIndexShardComponent {
private final ESLogger queryLogger;
private final ESLogger fetchLogger;
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN = "index.search.slowlog.threshold.query.warn";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO = "index.search.slowlog.threshold.query.info";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG = "index.search.slowlog.threshold.query.debug";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE = "index.search.slowlog.threshold.query.trace";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN = "index.search.slowlog.threshold.fetch.warn";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO = "index.search.slowlog.threshold.fetch.info";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG = "index.search.slowlog.threshold.fetch.debug";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE = "index.search.slowlog.threshold.fetch.trace";
public static final String INDEX_SEARCH_SLOWLOG_REFORMAT = "index.search.slowlog.reformat";
public static final String INDEX_SEARCH_SLOWLOG_LEVEL = "index.search.slowlog.level";
private static final String INDEX_SEARCH_SLOWLOG_PREFIX = "index.search.slowlog";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.warn";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.info";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.debug";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.trace";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.warn";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.info";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.debug";
public static final String INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE = INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.trace";
public static final String INDEX_SEARCH_SLOWLOG_REFORMAT = INDEX_SEARCH_SLOWLOG_PREFIX + ".reformat";
public static final String INDEX_SEARCH_SLOWLOG_LEVEL = INDEX_SEARCH_SLOWLOG_PREFIX + ".level";
class ApplySettings implements IndexSettingsService.Listener {
@Override
public synchronized void onRefreshSettings(Settings settings) {
long queryWarnThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.queryWarnThreshold)).nanos();
if (queryWarnThreshold != ShardSlowLogSearchService.this.queryWarnThreshold) {
ShardSlowLogSearchService.this.queryWarnThreshold = queryWarnThreshold;
}
long queryInfoThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.queryInfoThreshold)).nanos();
if (queryInfoThreshold != ShardSlowLogSearchService.this.queryInfoThreshold) {
ShardSlowLogSearchService.this.queryInfoThreshold = queryInfoThreshold;
}
long queryDebugThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.queryDebugThreshold)).nanos();
if (queryDebugThreshold != ShardSlowLogSearchService.this.queryDebugThreshold) {
ShardSlowLogSearchService.this.queryDebugThreshold = queryDebugThreshold;
}
long queryTraceThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.queryTraceThreshold)).nanos();
if (queryTraceThreshold != ShardSlowLogSearchService.this.queryTraceThreshold) {
ShardSlowLogSearchService.this.queryTraceThreshold = queryTraceThreshold;
}
long fetchWarnThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.fetchWarnThreshold)).nanos();
if (fetchWarnThreshold != ShardSlowLogSearchService.this.fetchWarnThreshold) {
ShardSlowLogSearchService.this.fetchWarnThreshold = fetchWarnThreshold;
}
long fetchInfoThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.fetchInfoThreshold)).nanos();
if (fetchInfoThreshold != ShardSlowLogSearchService.this.fetchInfoThreshold) {
ShardSlowLogSearchService.this.fetchInfoThreshold = fetchInfoThreshold;
}
long fetchDebugThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.fetchDebugThreshold)).nanos();
if (fetchDebugThreshold != ShardSlowLogSearchService.this.fetchDebugThreshold) {
ShardSlowLogSearchService.this.fetchDebugThreshold = fetchDebugThreshold;
}
long fetchTraceThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE, TimeValue.timeValueNanos(ShardSlowLogSearchService.this.fetchTraceThreshold)).nanos();
if (fetchTraceThreshold != ShardSlowLogSearchService.this.fetchTraceThreshold) {
ShardSlowLogSearchService.this.fetchTraceThreshold = fetchTraceThreshold;
}
String level = settings.get(INDEX_SEARCH_SLOWLOG_LEVEL, ShardSlowLogSearchService.this.level);
if (!level.equals(ShardSlowLogSearchService.this.level)) {
ShardSlowLogSearchService.this.queryLogger.setLevel(level.toUpperCase(Locale.ROOT));
ShardSlowLogSearchService.this.fetchLogger.setLevel(level.toUpperCase(Locale.ROOT));
ShardSlowLogSearchService.this.level = level;
}
boolean reformat = settings.getAsBoolean(INDEX_SEARCH_SLOWLOG_REFORMAT, ShardSlowLogSearchService.this.reformat);
if (reformat != ShardSlowLogSearchService.this.reformat) {
ShardSlowLogSearchService.this.reformat = reformat;
}
}
}
@Inject
public ShardSlowLogSearchService(ShardId shardId, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) {
super(shardId, indexSettings);
SearchSlowLog(Settings indexSettings) {
this.reformat = indexSettings.getAsBoolean(INDEX_SEARCH_SLOWLOG_REFORMAT, true);
@ -137,16 +80,14 @@ public class ShardSlowLogSearchService extends AbstractIndexShardComponent {
this.level = indexSettings.get(INDEX_SEARCH_SLOWLOG_LEVEL, "TRACE").toUpperCase(Locale.ROOT);
this.queryLogger = Loggers.getLogger(logger, ".query");
this.fetchLogger = Loggers.getLogger(logger, ".fetch");
this.queryLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query");
this.fetchLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch");
queryLogger.setLevel(level);
fetchLogger.setLevel(level);
indexSettingsService.addListener(new ApplySettings());
}
public void onQueryPhase(SearchContext context, long tookInNanos) {
void onQueryPhase(SearchContext context, long tookInNanos) {
if (queryWarnThreshold >= 0 && tookInNanos > queryWarnThreshold) {
queryLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
} else if (queryInfoThreshold >= 0 && tookInNanos > queryInfoThreshold) {
@ -158,7 +99,7 @@ public class ShardSlowLogSearchService extends AbstractIndexShardComponent {
}
}
public void onFetchPhase(SearchContext context, long tookInNanos) {
void onFetchPhase(SearchContext context, long tookInNanos) {
if (fetchWarnThreshold >= 0 && tookInNanos > fetchWarnThreshold) {
fetchLogger.warn("{}", new SlowLogSearchContextPrinter(context, tookInNanos, reformat));
} else if (fetchInfoThreshold >= 0 && tookInNanos > fetchInfoThreshold) {
@ -170,7 +111,55 @@ public class ShardSlowLogSearchService extends AbstractIndexShardComponent {
}
}
public static class SlowLogSearchContextPrinter {
synchronized void onRefreshSettings(Settings settings) {
long queryWarnThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN, TimeValue.timeValueNanos(this.queryWarnThreshold)).nanos();
if (queryWarnThreshold != this.queryWarnThreshold) {
this.queryWarnThreshold = queryWarnThreshold;
}
long queryInfoThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO, TimeValue.timeValueNanos(this.queryInfoThreshold)).nanos();
if (queryInfoThreshold != this.queryInfoThreshold) {
this.queryInfoThreshold = queryInfoThreshold;
}
long queryDebugThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG, TimeValue.timeValueNanos(this.queryDebugThreshold)).nanos();
if (queryDebugThreshold != this.queryDebugThreshold) {
this.queryDebugThreshold = queryDebugThreshold;
}
long queryTraceThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE, TimeValue.timeValueNanos(this.queryTraceThreshold)).nanos();
if (queryTraceThreshold != this.queryTraceThreshold) {
this.queryTraceThreshold = queryTraceThreshold;
}
long fetchWarnThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN, TimeValue.timeValueNanos(this.fetchWarnThreshold)).nanos();
if (fetchWarnThreshold != this.fetchWarnThreshold) {
this.fetchWarnThreshold = fetchWarnThreshold;
}
long fetchInfoThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO, TimeValue.timeValueNanos(this.fetchInfoThreshold)).nanos();
if (fetchInfoThreshold != this.fetchInfoThreshold) {
this.fetchInfoThreshold = fetchInfoThreshold;
}
long fetchDebugThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG, TimeValue.timeValueNanos(this.fetchDebugThreshold)).nanos();
if (fetchDebugThreshold != this.fetchDebugThreshold) {
this.fetchDebugThreshold = fetchDebugThreshold;
}
long fetchTraceThreshold = settings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE, TimeValue.timeValueNanos(this.fetchTraceThreshold)).nanos();
if (fetchTraceThreshold != this.fetchTraceThreshold) {
this.fetchTraceThreshold = fetchTraceThreshold;
}
String level = settings.get(INDEX_SEARCH_SLOWLOG_LEVEL, this.level);
if (!level.equals(this.level)) {
this.queryLogger.setLevel(level.toUpperCase(Locale.ROOT));
this.fetchLogger.setLevel(level.toUpperCase(Locale.ROOT));
this.level = level;
}
boolean reformat = settings.getAsBoolean(INDEX_SEARCH_SLOWLOG_REFORMAT, this.reformat);
if (reformat != this.reformat) {
this.reformat = reformat;
}
}
private static class SlowLogSearchContextPrinter {
private final SearchContext context;
private final long tookInNanos;
private final boolean reformat;

23
core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchService.java → core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
View File

@ -21,15 +21,10 @@ package org.elasticsearch.index.search.stats;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.search.internal.SearchContext;
import java.util.HashMap;
@ -38,19 +33,15 @@ import java.util.concurrent.TimeUnit;
/**
*/
public class ShardSearchService extends AbstractIndexShardComponent {
private final ShardSlowLogSearchService slowLogSearchService;
public final class ShardSearchStats {
private final SearchSlowLog slowLogSearchService;
private final StatsHolder totalStats = new StatsHolder();
private final CounterMetric openContexts = new CounterMetric();
private volatile Map<String, StatsHolder> groupsStats = ImmutableMap.of();
@Inject
public ShardSearchService(ShardId shardId, @IndexSettings Settings indexSettings, ShardSlowLogSearchService slowLogSearchService) {
super(shardId, indexSettings);
this.slowLogSearchService = slowLogSearchService;
public ShardSearchStats(Settings indexSettings) {
this.slowLogSearchService = new SearchSlowLog(indexSettings);
}
/**
@ -178,7 +169,11 @@ public class ShardSearchService extends AbstractIndexShardComponent {
openContexts.dec();
}
static class StatsHolder {
public void onRefreshSettings(Settings settings) {
slowLogSearchService.onRefreshSettings(settings);
}
final static class StatsHolder {
public final MeanMetric queryMetric = new MeanMetric();
public final MeanMetric fetchMetric = new MeanMetric();
public final CounterMetric queryCurrent = new CounterMetric();

36
core/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java
View File

@ -31,8 +31,8 @@ import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.gateway.GatewayAllocator;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService;
import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
import org.elasticsearch.index.indexing.IndexingSlowLog;
import org.elasticsearch.index.search.stats.SearchSlowLog;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.MergePolicyConfig;
import org.elasticsearch.index.store.IndexStore;
@ -78,22 +78,22 @@ public class IndexDynamicSettingsModule extends AbstractModule {
indexDynamicSettings.addDynamicSetting(EngineConfig.INDEX_GC_DELETES_SETTING, Validator.TIME);
indexDynamicSettings.addDynamicSetting(IndexShard.INDEX_FLUSH_ON_CLOSE, Validator.BOOLEAN);
indexDynamicSettings.addDynamicSetting(EngineConfig.INDEX_VERSION_MAP_SIZE, Validator.BYTES_SIZE_OR_PERCENTAGE);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_REFORMAT);
indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_LEVEL);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_REFORMAT);
indexDynamicSettings.addDynamicSetting(ShardSlowLogSearchService.INDEX_SEARCH_SLOWLOG_LEVEL);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT);
indexDynamicSettings.addDynamicSetting(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE, Validator.TIME);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_REFORMAT);
indexDynamicSettings.addDynamicSetting(SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL);
indexDynamicSettings.addDynamicSetting(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE, Validator.INTEGER);
indexDynamicSettings.addDynamicSetting(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED, Validator.DOUBLE);
indexDynamicSettings.addDynamicSetting(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT, Validator.BYTES_SIZE);

2
core/src/main/java/org/elasticsearch/index/gateway/CommitPoint.java → core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.gateway;
package org.elasticsearch.index.shard;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.Nullable;

2
core/src/main/java/org/elasticsearch/index/gateway/CommitPoints.java → core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.gateway;
package org.elasticsearch.index.shard;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

60
core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.RestoreSource;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -51,7 +52,6 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.VersionType;
@ -59,7 +59,6 @@ import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache;
import org.elasticsearch.index.cache.filter.FilterCacheStats;
import org.elasticsearch.index.cache.filter.ShardFilterCache;
import org.elasticsearch.index.cache.query.ShardQueryCache;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy;
@ -81,7 +80,7 @@ import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.SearchStats;
import org.elasticsearch.index.search.stats.ShardSearchService;
import org.elasticsearch.index.search.stats.ShardSearchStats;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.Store;
@ -100,6 +99,7 @@ import org.elasticsearch.index.warmer.WarmerStats;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.InternalIndicesLifecycle;
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat;
import org.elasticsearch.search.suggest.completion.CompletionStats;
@ -131,10 +131,9 @@ public class IndexShard extends AbstractIndexShardComponent {
private final MergeSchedulerConfig mergeSchedulerConfig;
private final IndexAliasesService indexAliasesService;
private final ShardIndexingService indexingService;
private final ShardSearchService searchService;
private final ShardSearchStats searchService;
private final ShardGetService getService;
private final ShardIndexWarmerService shardWarmerService;
private final ShardFilterCache shardFilterCache;
private final ShardQueryCache shardQueryCache;
private final ShardFieldData shardFieldData;
private final PercolatorQueriesRegistry percolatorQueriesRegistry;
@ -148,7 +147,6 @@ public class IndexShard extends AbstractIndexShardComponent {
private final Object mutex = new Object();
private final String checkIndexOnStartup;
private final NodeEnvironment nodeEnv;
private final CodecService codecService;
private final IndicesWarmer warmer;
private final SnapshotDeletionPolicy deletionPolicy;
@ -156,6 +154,8 @@ public class IndexShard extends AbstractIndexShardComponent {
private final EngineConfig engineConfig;
private final TranslogConfig translogConfig;
private final MergePolicyConfig mergePolicyConfig;
private final IndicesFilterCache indicesFilterCache;
private final StoreRecoveryService storeRecoveryService;
private TimeValue refreshInterval;
@ -189,13 +189,12 @@ public class IndexShard extends AbstractIndexShardComponent {
private final IndexShardOperationCounter indexShardOperationCounter;
@Inject
public IndexShard(ShardId shardId, IndexSettingsService indexSettingsService, IndicesLifecycle indicesLifecycle, Store store,
ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, ShardIndexingService indexingService, ShardSearchService searchService, ShardIndexWarmerService shardWarmerService,
ShardFilterCache shardFilterCache, ShardFieldData shardFieldData, PercolatorQueriesRegistry percolatorQueriesRegistry, ShardPercolateService shardPercolateService, CodecService codecService,
public IndexShard(ShardId shardId, IndexSettingsService indexSettingsService, IndicesLifecycle indicesLifecycle, Store store, StoreRecoveryService storeRecoveryService,
ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService,
IndicesFilterCache indicesFilterCache, ShardPercolateService shardPercolateService, CodecService codecService,
ShardTermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, IndexService indexService,
ShardQueryCache shardQueryCache, ShardBitsetFilterCache shardBitsetFilterCache,
@Nullable IndicesWarmer warmer, SnapshotDeletionPolicy deletionPolicy, SimilarityService similarityService, EngineFactory factory,
ClusterService clusterService, NodeEnvironment nodeEnv, ShardPath path, BigArrays bigArrays) {
ClusterService clusterService, ShardPath path, BigArrays bigArrays) {
super(shardId, indexSettingsService.getSettings());
this.codecService = codecService;
this.warmer = warmer;
@ -207,31 +206,31 @@ public class IndexShard extends AbstractIndexShardComponent {
this.indicesLifecycle = (InternalIndicesLifecycle) indicesLifecycle;
this.indexSettingsService = indexSettingsService;
this.store = store;
this.storeRecoveryService = storeRecoveryService;
this.mergeSchedulerConfig = new MergeSchedulerConfig(indexSettings);
this.threadPool = threadPool;
this.mapperService = mapperService;
this.queryParserService = queryParserService;
this.indexCache = indexCache;
this.indexAliasesService = indexAliasesService;
this.indexingService = indexingService;
this.indexingService = new ShardIndexingService(shardId, indexSettings);
this.getService = new ShardGetService(this, mapperService);
this.termVectorsService = termVectorsService.setIndexShard(this);
this.searchService = searchService;
this.shardWarmerService = shardWarmerService;
this.shardFilterCache = shardFilterCache;
this.shardQueryCache = shardQueryCache;
this.shardFieldData = shardFieldData;
this.percolatorQueriesRegistry = percolatorQueriesRegistry;
this.searchService = new ShardSearchStats(indexSettings);
this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings);
this.indicesFilterCache = indicesFilterCache;
this.shardQueryCache = new ShardQueryCache(shardId, indexSettings);
this.shardFieldData = new ShardFieldData();
this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryParserService, indexingService, indicesLifecycle, mapperService, indexFieldDataService, shardPercolateService);
this.shardPercolateService = shardPercolateService;
this.indexFieldDataService = indexFieldDataService;
this.indexService = indexService;
this.shardBitsetFilterCache = shardBitsetFilterCache;
this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings);
assert clusterService.localNode() != null : "Local node is null lifecycle state is: " + clusterService.lifecycleState();
this.localNode = clusterService.localNode();
state = IndexShardState.CREATED;
this.refreshInterval = indexSettings.getAsTime(INDEX_REFRESH_INTERVAL, EngineConfig.DEFAULT_REFRESH_INTERVAL);
this.flushOnClose = indexSettings.getAsBoolean(INDEX_FLUSH_ON_CLOSE, true);
this.nodeEnv = nodeEnv;
indexSettingsService.addListener(applyRefreshSettings);
this.path = path;
this.mergePolicyConfig = new MergePolicyConfig(logger, indexSettings);
@ -289,7 +288,7 @@ public class IndexShard extends AbstractIndexShardComponent {
return indexService;
}
public ShardSearchService searchService() {
public ShardSearchStats searchService() {
return this.searchService;
}
@ -297,10 +296,6 @@ public class IndexShard extends AbstractIndexShardComponent {
return this.shardWarmerService;
}
public ShardFilterCache filterCache() {
return this.shardFilterCache;
}
public ShardQueryCache queryCache() {
return this.shardQueryCache;
}
@ -620,7 +615,7 @@ public class IndexShard extends AbstractIndexShardComponent {
}
public FilterCacheStats filterCacheStats() {
return shardFilterCache.stats();
return indicesFilterCache.getStats(shardId);
}
public FieldDataStats fieldDataStats(String... fields) {
@ -760,7 +755,7 @@ public class IndexShard extends AbstractIndexShardComponent {
engine.flushAndClose();
}
} finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
IOUtils.close(engine);
IOUtils.close(engine, percolatorQueriesRegistry);
}
}
}
@ -1018,6 +1013,13 @@ public class IndexShard extends AbstractIndexShardComponent {
return path;
}
public void recoverFromStore(IndexShardRoutingTable shardRoutingTable, StoreRecoveryService.RecoveryListener recoveryListener) {
// we are the first primary, recover from the gateway
// if its post api allocation, the index should exists
final boolean shouldExist = shardRoutingTable.primaryAllocatedPostApi();
storeRecoveryService.recover(this, shouldExist, recoveryListener);
}
private class ApplyRefreshSettings implements IndexSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
@ -1101,6 +1103,8 @@ public class IndexShard extends AbstractIndexShardComponent {
}
}
mergePolicyConfig.onRefreshSettings(settings);
searchService.onRefreshSettings(settings);
indexingService.onRefreshSettings(settings);
if (change) {
refresh("apply settings");
}
@ -1340,7 +1344,7 @@ public class IndexShard extends AbstractIndexShardComponent {
}
};
return new EngineConfig(shardId,
threadPool, indexingService, indexSettingsService, warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
threadPool, indexingService, indexSettingsService.indexSettings(), warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.filter(), indexCache.filterPolicy(), translogConfig);
}

31
core/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java
View File

@ -22,24 +22,11 @@ package org.elasticsearch.index.shard;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache;
import org.elasticsearch.index.cache.filter.ShardFilterCache;
import org.elasticsearch.index.cache.query.ShardQueryCache;
import org.elasticsearch.index.engine.EngineFactory;
import org.elasticsearch.index.engine.InternalEngineFactory;
import org.elasticsearch.index.fielddata.ShardFieldData;
import org.elasticsearch.index.gateway.IndexShardGateway;
import org.elasticsearch.index.gateway.IndexShardGatewayService;
import org.elasticsearch.index.indexing.ShardIndexingService;
import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService;
import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
import org.elasticsearch.index.percolator.stats.ShardPercolateService;
import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
import org.elasticsearch.index.search.stats.ShardSearchService;
import org.elasticsearch.index.snapshots.IndexShardSnapshotAndRestoreService;
import org.elasticsearch.index.termvectors.ShardTermVectorsService;
import org.elasticsearch.index.translog.TranslogService;
import org.elasticsearch.index.warmer.ShardIndexWarmerService;
/**
* The {@code IndexShardModule} module is responsible for binding the correct
@ -57,11 +44,9 @@ public class IndexShardModule extends AbstractModule {
private final ShardId shardId;
private final Settings settings;
private final boolean primary;
private final ShardFilterCache shardFilterCache;
public IndexShardModule(ShardId shardId, boolean primary, Settings settings, ShardFilterCache shardFilterCache) {
public IndexShardModule(ShardId shardId, boolean primary, Settings settings) {
this.settings = settings;
this.shardFilterCache = shardFilterCache;
this.shardId = shardId;
this.primary = primary;
if (settings.get("index.translog.type") != null) {
@ -85,21 +70,9 @@ public class IndexShardModule extends AbstractModule {
}
bind(EngineFactory.class).to(settings.getAsClass(ENGINE_FACTORY, DEFAULT_ENGINE_FACTORY_CLASS, ENGINE_PREFIX, ENGINE_SUFFIX));
bind(ShardIndexWarmerService.class).asEagerSingleton();
bind(ShardIndexingService.class).asEagerSingleton();
bind(ShardSlowLogIndexingService.class).asEagerSingleton();
bind(ShardSearchService.class).asEagerSingleton();
bind(ShardSlowLogSearchService.class).asEagerSingleton();
bind(ShardFilterCache.class).toInstance(shardFilterCache);
bind(ShardQueryCache.class).asEagerSingleton();
bind(ShardBitsetFilterCache.class).asEagerSingleton();
bind(ShardFieldData.class).asEagerSingleton();
bind(IndexShardGateway.class).asEagerSingleton();
bind(IndexShardGatewayService.class).asEagerSingleton();
bind(PercolatorQueriesRegistry.class).asEagerSingleton();
bind(StoreRecoveryService.class).asEagerSingleton();
bind(ShardPercolateService.class).asEagerSingleton();
bind(ShardTermVectorsService.class).asEagerSingleton();
bind(IndexShardSnapshotAndRestoreService.class).asEagerSingleton();
}

10
core/src/main/java/org/elasticsearch/index/gateway/IndexShardGatewaySnapshotFailedException.java → core/src/main/java/org/elasticsearch/index/shard/IndexShardRecoveryException.java
View File

@ -17,16 +17,16 @@
* under the License.
*/
package org.elasticsearch.index.gateway;
package org.elasticsearch.index.shard;
import org.elasticsearch.index.shard.IndexShardException;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
public class IndexShardGatewaySnapshotFailedException extends IndexShardGatewayException {
public IndexShardGatewaySnapshotFailedException(ShardId shardId, String msg, Throwable cause) {
public class IndexShardRecoveryException extends IndexShardException {
public IndexShardRecoveryException(ShardId shardId, String msg, Throwable cause) {
super(shardId, msg, cause);
}
}
}

31
core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
View File

@ -23,34 +23,26 @@ import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache;
import org.elasticsearch.index.cache.filter.ShardFilterCache;
import org.elasticsearch.index.cache.query.ShardQueryCache;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.engine.EngineFactory;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fielddata.ShardFieldData;
import org.elasticsearch.index.indexing.ShardIndexingService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
import org.elasticsearch.index.percolator.stats.ShardPercolateService;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.search.stats.ShardSearchService;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.termvectors.ShardTermVectorsService;
import org.elasticsearch.index.warmer.ShardIndexWarmerService;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
@ -65,27 +57,22 @@ public final class ShadowIndexShard extends IndexShard {
@Inject
public ShadowIndexShard(ShardId shardId, IndexSettingsService indexSettingsService,
IndicesLifecycle indicesLifecycle, Store store,
IndicesLifecycle indicesLifecycle, Store store, StoreRecoveryService storeRecoveryService,
ThreadPool threadPool, MapperService mapperService,
IndexQueryParserService queryParserService, IndexCache indexCache,
IndexAliasesService indexAliasesService, ShardIndexingService indexingService,
ShardSearchService searchService,
ShardIndexWarmerService shardWarmerService, ShardFilterCache shardFilterCache,
ShardFieldData shardFieldData, PercolatorQueriesRegistry percolatorQueriesRegistry,
IndexAliasesService indexAliasesService, IndicesFilterCache indicesFilterCache,
ShardPercolateService shardPercolateService, CodecService codecService,
ShardTermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService,
IndexService indexService, ShardQueryCache shardQueryCache,
ShardBitsetFilterCache shardBitsetFilterCache, @Nullable IndicesWarmer warmer,
IndexService indexService, @Nullable IndicesWarmer warmer,
SnapshotDeletionPolicy deletionPolicy, SimilarityService similarityService,
EngineFactory factory, ClusterService clusterService,
NodeEnvironment nodeEnv, ShardPath path, BigArrays bigArrays) throws IOException {
super(shardId, indexSettingsService, indicesLifecycle, store,
ShardPath path, BigArrays bigArrays) throws IOException {
super(shardId, indexSettingsService, indicesLifecycle, store, storeRecoveryService,
threadPool, mapperService, queryParserService, indexCache, indexAliasesService,
indexingService, searchService, shardWarmerService, shardFilterCache,
shardFieldData, percolatorQueriesRegistry, shardPercolateService, codecService,
indicesFilterCache, shardPercolateService, codecService,
termVectorsService, indexFieldDataService, indexService,
shardQueryCache, shardBitsetFilterCache, warmer, deletionPolicy, similarityService,
factory, clusterService, nodeEnv, path, bigArrays);
warmer, deletionPolicy, similarityService,
factory, clusterService, path, bigArrays);
}
/**

3
core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
View File

@ -92,7 +92,8 @@ public final class ShardPath {
ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
if (load != null) {
if ((load.indexUUID.equals(indexUUID) || IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID)) == false) {
throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " excepted: " + indexUUID + " on shard path: " + path);
logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " expected: " + indexUUID + " on shard path: " + path);
}
if (loadedPath == null) {
loadedPath = path;

2
core/src/main/java/org/elasticsearch/index/gateway/SnapshotStatus.java → core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.gateway;
package org.elasticsearch.index.shard;
/**
*

337
core/src/main/java/org/elasticsearch/index/shard/StoreRecoveryService.java Normal file
View File

@ -0,0 +1,337 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.Directory;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.cluster.routing.RestoreSource;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.CancellableThreads;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.mapper.Mapping;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.snapshots.RestoreService;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
/**
*
*/
public class StoreRecoveryService extends AbstractIndexShardComponent implements Closeable {
private final MappingUpdatedAction mappingUpdatedAction;
private final ThreadPool threadPool;
private final ClusterService clusterService;
private final TimeValue waitForMappingUpdatePostRecovery;
private final CancellableThreads cancellableThreads = new CancellableThreads();
private static final String SETTING_MAPPING_UPDATE_WAIT_LEGACY = "index.gateway.wait_for_mapping_update_post_recovery";
private static final String SETTING_MAPPING_UPDATE_WAIT = "index.shard.wait_for_mapping_update_post_recovery";
private final RestoreService restoreService;
private final RepositoriesService repositoriesService;
@Inject
public StoreRecoveryService(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool,
MappingUpdatedAction mappingUpdatedAction, ClusterService clusterService, RepositoriesService repositoriesService, RestoreService restoreService) {
super(shardId, indexSettings);
this.threadPool = threadPool;
this.mappingUpdatedAction = mappingUpdatedAction;
this.restoreService = restoreService;
this.repositoriesService = repositoriesService;
this.clusterService = clusterService;
this.waitForMappingUpdatePostRecovery = indexSettings.getAsTime(SETTING_MAPPING_UPDATE_WAIT, indexSettings.getAsTime(SETTING_MAPPING_UPDATE_WAIT_LEGACY, TimeValue.timeValueSeconds(15)));
}
public interface RecoveryListener {
void onRecoveryDone();
void onIgnoreRecovery(String reason);
void onRecoveryFailed(IndexShardRecoveryException e);
}
/**
* Recovers the state of the shard from the gateway.
*/
public void recover(final IndexShard indexShard, final boolean indexShouldExists, final RecoveryListener listener) throws IndexShardRecoveryException {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if (!indexShard.routingEntry().primary()) {
listener.onRecoveryFailed(new IndexShardRecoveryException(shardId, "Trying to recover when the shard is in backup state", null));
return;
}
try {
if (indexShard.routingEntry().restoreSource() != null) {
indexShard.recovering("from snapshot", RecoveryState.Type.SNAPSHOT, indexShard.routingEntry().restoreSource());
} else {
indexShard.recovering("from store", RecoveryState.Type.STORE, clusterService.localNode());
}
} catch (IllegalIndexShardStateException e) {
// that's fine, since we might be called concurrently, just ignore this, we are already recovering
listener.onIgnoreRecovery("already in recovering process, " + e.getMessage());
return;
}
threadPool.generic().execute(new Runnable() {
@Override
public void run() {
try {
final RecoveryState recoveryState = indexShard.recoveryState();
if (indexShard.routingEntry().restoreSource() != null) {
logger.debug("restoring from {} ...", indexShard.routingEntry().restoreSource());
restore(indexShard, recoveryState);
} else {
logger.debug("starting recovery from shard_store ...");
recoverFromStore(indexShard, indexShouldExists, recoveryState);
}
// Check that the gateway didn't leave the shard in init or recovering stage. it is up to the gateway
// to call post recovery.
IndexShardState shardState = indexShard.state();
assert shardState != IndexShardState.CREATED && shardState != IndexShardState.RECOVERING : "recovery process of " + shardId + " didn't get to post_recovery. shardState [" + shardState + "]";
if (logger.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append("recovery completed from ").append("shard_store").append(", took [").append(timeValueMillis(recoveryState.getTimer().time())).append("]\n");
RecoveryState.Index index = recoveryState.getIndex();
sb.append(" index : files [").append(index.totalFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.totalBytes())).append("], took[")
.append(TimeValue.timeValueMillis(index.time())).append("]\n");
sb.append(" : recovered_files [").append(index.recoveredFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.recoveredBytes())).append("]\n");
sb.append(" : reusing_files [").append(index.reusedFileCount()).append("] with total_size [")
.append(new ByteSizeValue(index.reusedBytes())).append("]\n");
sb.append(" verify_index : took [").append(TimeValue.timeValueMillis(recoveryState.getVerifyIndex().time())).append("], check_index [")
.append(timeValueMillis(recoveryState.getVerifyIndex().checkIndexTime())).append("]\n");
sb.append(" translog : number_of_operations [").append(recoveryState.getTranslog().recoveredOperations())
.append("], took [").append(TimeValue.timeValueMillis(recoveryState.getTranslog().time())).append("]");
logger.trace(sb.toString());
} else if (logger.isDebugEnabled()) {
logger.debug("recovery completed from [shard_store], took [{}]", timeValueMillis(recoveryState.getTimer().time()));
}
listener.onRecoveryDone();
} catch (IndexShardRecoveryException e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
if ((e.getCause() instanceof IndexShardClosedException) || (e.getCause() instanceof IndexShardNotStartedException)) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(e);
} catch (IndexShardClosedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (IndexShardNotStartedException e) {
listener.onIgnoreRecovery("shard closed");
} catch (Exception e) {
if (indexShard.state() == IndexShardState.CLOSED) {
// got closed on us, just ignore this recovery
listener.onIgnoreRecovery("shard closed");
return;
}
listener.onRecoveryFailed(new IndexShardRecoveryException(shardId, "failed recovery", e));
}
}
});
}
/**
* Recovers the state of the shard from the store.
*/
private void recoverFromStore(IndexShard indexShard, boolean indexShouldExists, RecoveryState recoveryState) throws IndexShardRecoveryException {
indexShard.prepareForIndexRecovery();
long version = -1;
final Map<String, Mapping> typesToUpdate;
SegmentInfos si = null;
final Store store = indexShard.store();
store.incRef();
try {
try {
store.failIfCorrupted();
try {
si = store.readLastCommittedSegmentsInfo();
} catch (Throwable e) {
String files = "_unknown_";
try {
files = Arrays.toString(store.directory().listAll());
} catch (Throwable e1) {
files += " (failure=" + ExceptionsHelper.detailedMessage(e1) + ")";
}
if (indexShouldExists) {
throw new IndexShardRecoveryException(shardId(), "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files, e);
}
}
if (si != null) {
if (indexShouldExists) {
version = si.getVersion();
} else {
// it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
// its a "new index create" API, we have to do something, so better to clean it than use same data
logger.trace("cleaning existing shard, shouldn't exists");
IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
writer.close();
recoveryState.getTranslog().totalOperations(0);
}
}
} catch (Throwable e) {
throw new IndexShardRecoveryException(shardId(), "failed to fetch index version after copying it over", e);
}
recoveryState.getIndex().updateVersion(version);
// since we recover from local, just fill the files and size
try {
final RecoveryState.Index index = recoveryState.getIndex();
if (si != null) {
final Directory directory = store.directory();
for (String name : Lucene.files(si)) {
long length = directory.fileLength(name);
index.addFileDetail(name, length, true);
}
}
} catch (IOException e) {
logger.debug("failed to list file details", e);
}
if (indexShouldExists == false) {
recoveryState.getTranslog().totalOperations(0);
recoveryState.getTranslog().totalOperationsOnStart(0);
}
typesToUpdate = indexShard.performTranslogRecovery();
indexShard.finalizeRecovery();
String indexName = indexShard.shardId().index().name();
for (Map.Entry<String, Mapping> entry : typesToUpdate.entrySet()) {
validateMappingUpdate(indexName, entry.getKey(), entry.getValue());
}
indexShard.postRecovery("post recovery from shard_store");
} catch (EngineException e) {
throw new IndexShardRecoveryException(shardId, "failed to recovery from gateway", e);
} finally {
store.decRef();
}
}
private void validateMappingUpdate(final String indexName, final String type, Mapping update) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> error = new AtomicReference<>();
mappingUpdatedAction.updateMappingOnMaster(indexName, type, update, waitForMappingUpdatePostRecovery, new MappingUpdatedAction.MappingUpdateListener() {
@Override
public void onMappingUpdate() {
latch.countDown();
}
@Override
public void onFailure(Throwable t) {
latch.countDown();
error.set(t);
}
});
cancellableThreads.execute(new CancellableThreads.Interruptable() {
@Override
public void run() throws InterruptedException {
try {
if (latch.await(waitForMappingUpdatePostRecovery.millis(), TimeUnit.MILLISECONDS) == false) {
logger.debug("waited for mapping update on master for [{}], yet timed out", type);
} else {
if (error.get() != null) {
throw new IndexShardRecoveryException(shardId, "Failed to propagate mappings on master post recovery", error.get());
}
}
} catch (InterruptedException e) {
logger.debug("interrupted while waiting for mapping update");
throw e;
}
}
});
}
/**
* Restores shard from {@link RestoreSource} associated with this shard in routing table
*
* @param recoveryState recovery state
*/
private void restore(final IndexShard indexShard, final RecoveryState recoveryState) {
RestoreSource restoreSource = indexShard.routingEntry().restoreSource();
if (restoreSource == null) {
throw new IndexShardRestoreFailedException(shardId, "empty restore source");
}
if (logger.isTraceEnabled()) {
logger.trace("[{}] restoring shard [{}]", restoreSource.snapshotId(), shardId);
}
try {
recoveryState.getTranslog().totalOperations(0);
recoveryState.getTranslog().totalOperationsOnStart(0);
indexShard.prepareForIndexRecovery();
IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
ShardId snapshotShardId = shardId;
if (!shardId.getIndex().equals(restoreSource.index())) {
snapshotShardId = new ShardId(restoreSource.index(), shardId.id());
}
indexShardRepository.restore(restoreSource.snapshotId(), shardId, snapshotShardId, recoveryState);
indexShard.skipTranslogRecovery(true);
indexShard.finalizeRecovery();
indexShard.postRecovery("restore done");
restoreService.indexShardRestoreCompleted(restoreSource.snapshotId(), shardId);
} catch (Throwable t) {
if (Lucene.isCorruptionException(t)) {
restoreService.failRestore(restoreSource.snapshotId(), shardId());
}
throw new IndexShardRestoreFailedException(shardId, "restore failed", t);
}
}
@Override
public void close() {
cancellableThreads.cancel("closed");
}
}

Some files were not shown because too many files have changed in this diff Show More