honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring

This commit is contained in:
javanna 2015-08-31 13:28:48 +02:00 committed by Luca Cavanna
parent f624e350fb 6e2dc73023
commit faf526dccb
169 changed files with 3280 additions and 1814 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/pom.xml
View File

@ -314,6 +314,7 @@
<include>org/elasticsearch/common/util/MockBigArrays.class</include>
<include>org/elasticsearch/common/util/MockBigArrays$*.class</include>
<include>org/elasticsearch/node/NodeMocksPlugin.class</include>
<include>org/elasticsearch/node/MockNode.class</include>
</includes>
<excludes>
<!-- unit tests for yaml suite parser & rest spec parser need to be excluded -->

5
core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.action.admin.cluster.stats;
import com.carrotsearch.hppc.ObjectObjectHashMap;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -67,10 +66,10 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
for (ClusterStatsNodeResponse r : nodeResponses) {
for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
ShardStats indexShardStats = countsPerIndex.get(shardStats.getIndex());
ShardStats indexShardStats = countsPerIndex.get(shardStats.getShardRouting().getIndex());
if (indexShardStats == null) {
indexShardStats = new ShardStats();
countsPerIndex.put(shardStats.getIndex(), indexShardStats);
countsPerIndex.put(shardStats.getShardRouting().getIndex(), indexShardStats);
}
indexShardStats.total++;

3
core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.cluster.health.ClusterIndexHealth;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.support.ActionFilters;
@ -106,7 +107,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
for (IndexShard indexShard : indexService) {
if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
// only report on fully started shards
shardsStats.add(new ShardStats(indexShard, SHARD_STATS_FLAGS));
shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
}
}
}

92
core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheRequest.java vendored
View File

@ -1,92 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.cache.clear;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
/**
*
*/
class ShardClearIndicesCacheRequest extends BroadcastShardRequest {
private boolean queryCache = false;
private boolean fieldDataCache = false;
private boolean recycler;
private boolean requestCache = false;
private String[] fields = null;
ShardClearIndicesCacheRequest() {
}
ShardClearIndicesCacheRequest(ShardId shardId, ClearIndicesCacheRequest request) {
super(shardId, request);
queryCache = request.queryCache();
fieldDataCache = request.fieldDataCache();
fields = request.fields();
recycler = request.recycler();
requestCache = request.requestCache();
}
public boolean queryCache() {
return queryCache;
}
public boolean requestCache() {
return requestCache;
}
public boolean fieldDataCache() {
return this.fieldDataCache;
}
public boolean recycler() {
return this.recycler;
}
public String[] fields() {
return this.fields;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
queryCache = in.readBoolean();
fieldDataCache = in.readBoolean();
recycler = in.readBoolean();
fields = in.readStringArray();
requestCache = in.readBoolean();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeBoolean(queryCache);
out.writeBoolean(fieldDataCache);
out.writeBoolean(recycler);
out.writeStringArrayNullable(fields);
out.writeBoolean(requestCache);
}
}

36
core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheResponse.java vendored
View File

@ -1,36 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.cache.clear;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
class ShardClearIndicesCacheResponse extends BroadcastShardResponse {
ShardClearIndicesCacheResponse() {
}
ShardClearIndicesCacheResponse(ShardId shardId) {
super(shardId);
}
}

59
core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java vendored
View File

@ -21,17 +21,16 @@ package org.elasticsearch.action.admin.indices.cache.clear;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
@ -40,14 +39,14 @@ import org.elasticsearch.indices.cache.request.IndicesRequestCache;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Indices clear cache action.
*/
public class TransportClearIndicesCacheAction extends TransportBroadcastAction<ClearIndicesCacheRequest, ClearIndicesCacheResponse, ShardClearIndicesCacheRequest, ShardClearIndicesCacheResponse> {
public class TransportClearIndicesCacheAction extends TransportBroadcastByNodeAction<ClearIndicesCacheRequest, ClearIndicesCacheResponse, TransportBroadcastByNodeAction.EmptyResult> {
private final IndicesService indicesService;
private final IndicesRequestCache indicesRequestCache;
@ -58,48 +57,33 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastAction<C
IndicesRequestCache indicesQueryCache, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, ClearIndicesCacheAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
ClearIndicesCacheRequest.class, ShardClearIndicesCacheRequest.class, ThreadPool.Names.MANAGEMENT);
ClearIndicesCacheRequest.class, ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
this.indicesRequestCache = indicesQueryCache;
}
@Override
protected ClearIndicesCacheResponse newResponse(ClearIndicesCacheRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// simply ignore non active shards
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
successfulShards++;
}
}
return new ClearIndicesCacheResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures);
protected EmptyResult readShardResult(StreamInput in) throws IOException {
return EmptyResult.readEmptyResultFrom(in);
}
@Override
protected ShardClearIndicesCacheRequest newShardRequest(int numShards, ShardRouting shard, ClearIndicesCacheRequest request) {
return new ShardClearIndicesCacheRequest(shard.shardId(), request);
protected ClearIndicesCacheResponse newResponse(ClearIndicesCacheRequest request, int totalShards, int successfulShards, int failedShards, List<EmptyResult> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new ClearIndicesCacheResponse(totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardClearIndicesCacheResponse newShardResponse() {
return new ShardClearIndicesCacheResponse();
protected ClearIndicesCacheRequest readRequestFrom(StreamInput in) throws IOException {
final ClearIndicesCacheRequest request = new ClearIndicesCacheRequest();
request.readFrom(in);
return request;
}
@Override
protected ShardClearIndicesCacheResponse shardOperation(ShardClearIndicesCacheRequest request) {
IndexService service = indicesService.indexService(request.shardId().getIndex());
protected EmptyResult shardOperation(ClearIndicesCacheRequest request, ShardRouting shardRouting) {
IndexService service = indicesService.indexService(shardRouting.getIndex());
if (service != null) {
IndexShard shard = service.shard(request.shardId().id());
IndexShard shard = service.shard(shardRouting.id());
boolean clearedAtLeastOne = false;
if (request.queryCache()) {
clearedAtLeastOne = true;
@ -137,15 +121,15 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastAction<C
}
}
}
return new ShardClearIndicesCacheResponse(request.shardId());
return EmptyResult.INSTANCE;
}
/**
* The refresh request works against *all* shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, ClearIndicesCacheRequest request, String[] concreteIndices) {
return clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, ClearIndicesCacheRequest request, String[] concreteIndices) {
return clusterState.routingTable().allShards(concreteIndices);
}
@Override
@ -157,5 +141,4 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastAction<C
protected ClusterBlockException checkRequestBlock(ClusterState state, ClearIndicesCacheRequest request, String[] concreteIndices) {
return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices);
}
}

60
core/src/main/java/org/elasticsearch/action/admin/indices/optimize/ShardOptimizeRequest.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.optimize;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
/**
*
*/
final class ShardOptimizeRequest extends BroadcastShardRequest {
private OptimizeRequest request = new OptimizeRequest();
ShardOptimizeRequest() {
}
ShardOptimizeRequest(ShardId shardId, OptimizeRequest request) {
super(shardId, request);
this.request = request;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
request.readFrom(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
request.writeTo(out);
}
public OptimizeRequest optimizeRequest() {
return this.request;
}
}

36
core/src/main/java/org/elasticsearch/action/admin/indices/optimize/ShardOptimizeResponse.java
View File

@ -1,36 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.optimize;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
class ShardOptimizeResponse extends BroadcastShardResponse {
ShardOptimizeResponse() {
}
ShardOptimizeResponse(ShardId shardId) {
super(shardId);
}
}

57
core/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java
View File

@ -21,31 +21,29 @@ package org.elasticsearch.action.admin.indices.optimize;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Optimize index/indices action.
*/
public class TransportOptimizeAction extends TransportBroadcastAction<OptimizeRequest, OptimizeResponse, ShardOptimizeRequest, ShardOptimizeResponse> {
public class TransportOptimizeAction extends TransportBroadcastByNodeAction<OptimizeRequest, OptimizeResponse, TransportBroadcastByNodeAction.EmptyResult> {
private final IndicesService indicesService;
@ -54,55 +52,40 @@ public class TransportOptimizeAction extends TransportBroadcastAction<OptimizeRe
TransportService transportService, IndicesService indicesService,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, OptimizeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
OptimizeRequest.class, ShardOptimizeRequest.class, ThreadPool.Names.OPTIMIZE);
OptimizeRequest.class, ThreadPool.Names.OPTIMIZE);
this.indicesService = indicesService;
}
@Override
protected OptimizeResponse newResponse(OptimizeRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// a non active shard, ignore...
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
successfulShards++;
}
}
return new OptimizeResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures);
protected EmptyResult readShardResult(StreamInput in) throws IOException {
return EmptyResult.readEmptyResultFrom(in);
}
@Override
protected ShardOptimizeRequest newShardRequest(int numShards, ShardRouting shard, OptimizeRequest request) {
return new ShardOptimizeRequest(shard.shardId(), request);
protected OptimizeResponse newResponse(OptimizeRequest request, int totalShards, int successfulShards, int failedShards, List<EmptyResult> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new OptimizeResponse(totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardOptimizeResponse newShardResponse() {
return new ShardOptimizeResponse();
protected OptimizeRequest readRequestFrom(StreamInput in) throws IOException {
final OptimizeRequest request = new OptimizeRequest();
request.readFrom(in);
return request;
}
@Override
protected ShardOptimizeResponse shardOperation(ShardOptimizeRequest request) {
IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).shardSafe(request.shardId().id());
indexShard.optimize(request.optimizeRequest());
return new ShardOptimizeResponse(request.shardId());
protected EmptyResult shardOperation(OptimizeRequest request, ShardRouting shardRouting) {
IndexShard indexShard = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()).shardSafe(shardRouting.shardId().id());
indexShard.optimize(request);
return EmptyResult.INSTANCE;
}
/**
* The refresh request works against *all* shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, OptimizeRequest request, String[] concreteIndices) {
return clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, OptimizeRequest request, String[] concreteIndices) {
return clusterState.routingTable().allShards(concreteIndices);
}
@Override

39
core/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.indices.recovery.RecoveryState;
import java.io.IOException;
import java.util.ArrayList;
@ -38,7 +39,7 @@ import java.util.Map;
public class RecoveryResponse extends BroadcastResponse implements ToXContent {
private boolean detailed = false;
private Map<String, List<ShardRecoveryResponse>> shardResponses = new HashMap<>();
private Map<String, List<RecoveryState>> shardRecoveryStates = new HashMap<>();
public RecoveryResponse() { }
@ -50,18 +51,18 @@ public class RecoveryResponse extends BroadcastResponse implements ToXContent {
* @param successfulShards Count of shards successfully processed
* @param failedShards Count of shards which failed to process
* @param detailed Display detailed metrics
* @param shardResponses Map of indices to shard recovery information
* @param shardRecoveryStates Map of indices to shard recovery information
* @param shardFailures List of failures processing shards
*/
public RecoveryResponse(int totalShards, int successfulShards, int failedShards, boolean detailed,
Map<String, List<ShardRecoveryResponse>> shardResponses, List<ShardOperationFailedException> shardFailures) {
Map<String, List<RecoveryState>> shardRecoveryStates, List<ShardOperationFailedException> shardFailures) {
super(totalShards, successfulShards, failedShards, shardFailures);
this.shardResponses = shardResponses;
this.shardRecoveryStates = shardRecoveryStates;
this.detailed = detailed;
}
public boolean hasRecoveries() {
return shardResponses.size() > 0;
return shardRecoveryStates.size() > 0;
}
public boolean detailed() {
@ -72,23 +73,23 @@ public class RecoveryResponse extends BroadcastResponse implements ToXContent {
this.detailed = detailed;
}
public Map<String, List<ShardRecoveryResponse>> shardResponses() {
return shardResponses;
public Map<String, List<RecoveryState>> shardRecoveryStates() {
return shardRecoveryStates;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (hasRecoveries()) {
for (String index : shardResponses.keySet()) {
List<ShardRecoveryResponse> responses = shardResponses.get(index);
if (responses == null || responses.size() == 0) {
for (String index : shardRecoveryStates.keySet()) {
List<RecoveryState> recoveryStates = shardRecoveryStates.get(index);
if (recoveryStates == null || recoveryStates.size() == 0) {
continue;
}
builder.startObject(index);
builder.startArray("shards");
for (ShardRecoveryResponse recoveryResponse : responses) {
for (RecoveryState recoveryState : recoveryStates) {
builder.startObject();
recoveryResponse.toXContent(builder, params);
recoveryState.toXContent(builder, params);
builder.endObject();
}
builder.endArray();
@ -101,12 +102,12 @@ public class RecoveryResponse extends BroadcastResponse implements ToXContent {
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeVInt(shardResponses.size());
for (Map.Entry<String, List<ShardRecoveryResponse>> entry : shardResponses.entrySet()) {
out.writeVInt(shardRecoveryStates.size());
for (Map.Entry<String, List<RecoveryState>> entry : shardRecoveryStates.entrySet()) {
out.writeString(entry.getKey());
out.writeVInt(entry.getValue().size());
for (ShardRecoveryResponse recoveryResponse : entry.getValue()) {
recoveryResponse.writeTo(out);
for (RecoveryState recoveryState : entry.getValue()) {
recoveryState.writeTo(out);
}
}
}
@ -118,11 +119,11 @@ public class RecoveryResponse extends BroadcastResponse implements ToXContent {
for (int i = 0; i < size; i++) {
String s = in.readString();
int listSize = in.readVInt();
List<ShardRecoveryResponse> list = new ArrayList<>(listSize);
List<RecoveryState> list = new ArrayList<>(listSize);
for (int j = 0; j < listSize; j++) {
list.add(ShardRecoveryResponse.readShardRecoveryResponse(in));
list.add(RecoveryState.readRecoveryState(in));
}
shardResponses.put(s, list);
shardRecoveryStates.put(s, list);
}
}
}

100
core/src/main/java/org/elasticsearch/action/admin/indices/recovery/ShardRecoveryResponse.java
View File

@ -1,100 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.recovery;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.recovery.RecoveryState;
import java.io.IOException;
/**
* Information regarding the recovery state of a shard.
*/
public class ShardRecoveryResponse extends BroadcastShardResponse implements ToXContent {
RecoveryState recoveryState;
public ShardRecoveryResponse() { }
/**
* Constructs shard recovery information for the given index and shard id.
*
* @param shardId Id of the shard
*/
ShardRecoveryResponse(ShardId shardId) {
super(shardId);
}
/**
* Sets the recovery state information for the shard.
*
* @param recoveryState Recovery state
*/
public void recoveryState(RecoveryState recoveryState) {
this.recoveryState = recoveryState;
}
/**
* Gets the recovery state information for the shard. Null if shard wasn't recovered / recovery didn't start yet.
*
* @return Recovery state
*/
@Nullable
public RecoveryState recoveryState() {
return recoveryState;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
recoveryState.toXContent(builder, params);
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
recoveryState.writeTo(out);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
recoveryState = RecoveryState.readRecoveryState(in);
}
/**
* Builds a new ShardRecoveryResponse from the give input stream.
*
* @param in Input stream
* @return A new ShardRecoveryResponse
* @throws IOException
*/
public static ShardRecoveryResponse readShardRecoveryResponse(StreamInput in) throws IOException {
ShardRecoveryResponse response = new ShardRecoveryResponse();
response.readFrom(in);
return response;
}
}

114
core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
View File

@ -19,40 +19,37 @@
package org.elasticsearch.action.admin.indices.recovery;
import com.google.common.collect.Maps;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Transport action for shard recovery operation. This transport action does not actually
* perform shard recovery, it only reports on recoveries (both active and complete).
*/
public class TransportRecoveryAction extends TransportBroadcastAction<RecoveryRequest, RecoveryResponse, TransportRecoveryAction.ShardRecoveryRequest, ShardRecoveryResponse> {
public class TransportRecoveryAction extends TransportBroadcastByNodeAction<RecoveryRequest, RecoveryResponse, RecoveryState> {
private final IndicesService indicesService;
@ -61,84 +58,55 @@ public class TransportRecoveryAction extends TransportBroadcastAction<RecoveryRe
TransportService transportService, IndicesService indicesService,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, RecoveryAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
RecoveryRequest.class, ShardRecoveryRequest.class, ThreadPool.Names.MANAGEMENT);
RecoveryRequest.class, ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
}
@Override
protected RecoveryResponse newResponse(RecoveryRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
protected RecoveryState readShardResult(StreamInput in) throws IOException {
return RecoveryState.readRecoveryState(in);
}
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
Map<String, List<ShardRecoveryResponse>> shardResponses = new HashMap<>();
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// simply ignore non active shards
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
@Override
protected RecoveryResponse newResponse(RecoveryRequest request, int totalShards, int successfulShards, int failedShards, List<RecoveryState> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
Map<String, List<RecoveryState>> shardResponses = Maps.newHashMap();
for (RecoveryState recoveryState : responses) {
if (recoveryState == null) {
continue;
}
String indexName = recoveryState.getShardId().getIndex();
if (!shardResponses.containsKey(indexName)) {
shardResponses.put(indexName, new ArrayList<RecoveryState>());
}
if (request.activeOnly()) {
if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
shardResponses.get(indexName).add(recoveryState);
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
ShardRecoveryResponse recoveryResponse = (ShardRecoveryResponse) shardResponse;
successfulShards++;
if (recoveryResponse.recoveryState() == null) {
// recovery not yet started
continue;
}
String indexName = recoveryResponse.getIndex();
List<ShardRecoveryResponse> responses = shardResponses.get(indexName);
if (responses == null) {
responses = new ArrayList<>();
shardResponses.put(indexName, responses);
}
if (request.activeOnly()) {
if (recoveryResponse.recoveryState().getStage() != RecoveryState.Stage.DONE) {
responses.add(recoveryResponse);
}
} else {
responses.add(recoveryResponse);
}
shardResponses.get(indexName).add(recoveryState);
}
}
return new RecoveryResponse(shardsResponses.length(), successfulShards,
failedShards, request.detailed(), shardResponses, shardFailures);
return new RecoveryResponse(totalShards, successfulShards, failedShards, request.detailed(), shardResponses, shardFailures);
}
@Override
protected ShardRecoveryRequest newShardRequest(int numShards, ShardRouting shard, RecoveryRequest request) {
return new ShardRecoveryRequest(shard.shardId(), request);
protected RecoveryRequest readRequestFrom(StreamInput in) throws IOException {
final RecoveryRequest recoveryRequest = new RecoveryRequest();
recoveryRequest.readFrom(in);
return recoveryRequest;
}
@Override
protected ShardRecoveryResponse newShardResponse() {
return new ShardRecoveryResponse();
protected RecoveryState shardOperation(RecoveryRequest request, ShardRouting shardRouting) {
IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
return indexShard.recoveryState();
}
@Override
protected ShardRecoveryResponse shardOperation(ShardRecoveryRequest request) {
IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(request.shardId().id());
ShardRecoveryResponse shardRecoveryResponse = new ShardRecoveryResponse(request.shardId());
RecoveryState state = indexShard.recoveryState();
shardRecoveryResponse.recoveryState(state);
return shardRecoveryResponse;
}
@Override
protected GroupShardsIterator shards(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
return state.routingTable().allAssignedShardsGrouped(concreteIndices, true, true);
protected ShardsIterator shards(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
return state.routingTable().allShardsIncludingRelocationTargets(concreteIndices);
}
@Override
@ -150,14 +118,4 @@ public class TransportRecoveryAction extends TransportBroadcastAction<RecoveryRe
protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices);
}
static class ShardRecoveryRequest extends BroadcastShardRequest {
ShardRecoveryRequest() {
}
ShardRecoveryRequest(ShardId shardId, RecoveryRequest request) {
super(shardId, request);
}
}
}

5
core/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
View File

@ -24,7 +24,6 @@ import com.google.common.collect.Sets;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.ByteSizeValue;
@ -50,7 +49,7 @@ public class IndicesSegmentResponse extends BroadcastResponse implements ToXCont
}
IndicesSegmentResponse(ShardSegments[] shards, ClusterState clusterState, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
IndicesSegmentResponse(ShardSegments[] shards, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
super(totalShards, successfulShards, failedShards, shardFailures);
this.shards = shards;
}
@ -63,7 +62,7 @@ public class IndicesSegmentResponse extends BroadcastResponse implements ToXCont
Set<String> indices = Sets.newHashSet();
for (ShardSegments shard : shards) {
indices.add(shard.getIndex());
indices.add(shard.getShardRouting().getIndex());
}
for (String index : indices) {

7
core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java
View File

@ -20,10 +20,10 @@
package org.elasticsearch.action.admin.indices.segments;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.index.engine.Segment;
import java.io.IOException;
@ -33,7 +33,7 @@ import java.util.List;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
public class ShardSegments extends BroadcastShardResponse implements Iterable<Segment> {
public class ShardSegments implements Streamable, Iterable<Segment> {
private ShardRouting shardRouting;
@ -43,7 +43,6 @@ public class ShardSegments extends BroadcastShardResponse implements Iterable<Se
}
ShardSegments(ShardRouting shardRouting, List<Segment> segments) {
super(shardRouting.shardId());
this.shardRouting = shardRouting;
this.segments = segments;
}
@ -89,7 +88,6 @@ public class ShardSegments extends BroadcastShardResponse implements Iterable<Se
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
shardRouting = readShardRoutingEntry(in);
int size = in.readVInt();
if (size == 0) {
@ -104,7 +102,6 @@ public class ShardSegments extends BroadcastShardResponse implements Iterable<Se
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
shardRouting.writeTo(out);
out.writeVInt(segments.size());
for (Segment segment : segments) {

85
core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
View File

@ -21,37 +21,30 @@ package org.elasticsearch.action.admin.indices.segments;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
*
*/
public class TransportIndicesSegmentsAction extends TransportBroadcastAction<IndicesSegmentsRequest, IndicesSegmentResponse, TransportIndicesSegmentsAction.IndexShardSegmentRequest, ShardSegments> {
public class TransportIndicesSegmentsAction extends TransportBroadcastByNodeAction<IndicesSegmentsRequest, IndicesSegmentResponse, ShardSegments> {
private final IndicesService indicesService;
@ -59,7 +52,7 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastAction<Ind
public TransportIndicesSegmentsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
IndicesService indicesService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, IndicesSegmentsAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
IndicesSegmentsRequest.class, TransportIndicesSegmentsAction.IndexShardSegmentRequest.class, ThreadPool.Names.MANAGEMENT);
IndicesSegmentsRequest.class, ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
}
@ -67,8 +60,8 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastAction<Ind
* Segments goes across *all* active shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, IndicesSegmentsRequest request, String[] concreteIndices) {
return clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, IndicesSegmentsRequest request, String[] concreteIndices) {
return clusterState.routingTable().allShards(concreteIndices);
}
@Override
@ -82,68 +75,26 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastAction<Ind
}
@Override
protected IndicesSegmentResponse newResponse(IndicesSegmentsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
final List<ShardSegments> shards = new ArrayList<>();
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// simply ignore non active shards
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
shards.add((ShardSegments) shardResponse);
successfulShards++;
}
}
return new IndicesSegmentResponse(shards.toArray(new ShardSegments[shards.size()]), clusterState, shardsResponses.length(), successfulShards, failedShards, shardFailures);
protected ShardSegments readShardResult(StreamInput in) throws IOException {
return ShardSegments.readShardSegments(in);
}
@Override
protected IndexShardSegmentRequest newShardRequest(int numShards, ShardRouting shard, IndicesSegmentsRequest request) {
return new IndexShardSegmentRequest(shard.shardId(), request);
protected IndicesSegmentResponse newResponse(IndicesSegmentsRequest request, int totalShards, int successfulShards, int failedShards, List<ShardSegments> results, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new IndicesSegmentResponse(results.toArray(new ShardSegments[results.size()]), totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardSegments newShardResponse() {
return new ShardSegments();
protected IndicesSegmentsRequest readRequestFrom(StreamInput in) throws IOException {
final IndicesSegmentsRequest request = new IndicesSegmentsRequest();
request.readFrom(in);
return request;
}
@Override
protected ShardSegments shardOperation(IndexShardSegmentRequest request) {
IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(request.shardId().id());
return new ShardSegments(indexShard.routingEntry(), indexShard.engine().segments(request.verbose));
}
static class IndexShardSegmentRequest extends BroadcastShardRequest {
boolean verbose;
IndexShardSegmentRequest() {
verbose = false;
}
IndexShardSegmentRequest(ShardId shardId, IndicesSegmentsRequest request) {
super(shardId, request);
verbose = request.verbose();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeBoolean(verbose);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
verbose = in.readBoolean();
}
protected ShardSegments shardOperation(IndicesSegmentsRequest request, ShardRouting shardRouting) {
IndexService indexService = indicesService.indexServiceSafe(shardRouting.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRouting.id());
return new ShardSegments(indexShard.routingEntry(), indexShard.engine().segments(request.verbose()));
}
}

5
core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
View File

@ -24,7 +24,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -51,7 +50,7 @@ public class IndicesStatsResponse extends BroadcastResponse implements ToXConten
}
IndicesStatsResponse(ShardStats[] shards, ClusterState clusterState, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
IndicesStatsResponse(ShardStats[] shards, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
super(totalShards, successfulShards, failedShards, shardFailures);
this.shards = shards;
}
@ -90,7 +89,7 @@ public class IndicesStatsResponse extends BroadcastResponse implements ToXConten
Set<String> indices = Sets.newHashSet();
for (ShardStats shard : shards) {
indices.add(shard.getIndex());
indices.add(shard.getShardRouting().getIndex());
}
for (String index : indices) {

22
core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
View File

@ -19,11 +19,11 @@
package org.elasticsearch.action.admin.indices.stats;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
@ -37,7 +37,7 @@ import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEnt
/**
*/
public class ShardStats extends BroadcastShardResponse implements ToXContent {
public class ShardStats implements Streamable, ToXContent {
private ShardRouting shardRouting;
private CommonStats commonStats;
@Nullable
@ -49,14 +49,13 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
ShardStats() {
}
public ShardStats(IndexShard indexShard, CommonStatsFlags flags) {
super(indexShard.shardId());
this.shardRouting = indexShard.routingEntry();
this.dataPath = indexShard.shardPath().getRootDataPath().toString();
this.statePath = indexShard.shardPath().getRootStatePath().toString();
this.isCustomDataPath = indexShard.shardPath().isCustomDataPath();
this.commonStats = new CommonStats(indexShard, flags);
this.commitStats = indexShard.commitStats();
public ShardStats(ShardRouting routing, ShardPath shardPath, CommonStats commonStats, CommitStats commitStats) {
this.shardRouting = routing;
this.dataPath = shardPath.getRootDataPath().toString();
this.statePath = shardPath.getRootStatePath().toString();
this.isCustomDataPath = shardPath.isCustomDataPath();
this.commitStats = commitStats;
this.commonStats = commonStats;
}
/**
@ -94,7 +93,6 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
shardRouting = readShardRoutingEntry(in);
commonStats = CommonStats.readCommonStats(in);
commitStats = CommitStats.readOptionalCommitStatsFrom(in);
@ -105,7 +103,6 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
shardRouting.writeTo(out);
commonStats.writeTo(out);
out.writeOptionalStreamable(commitStats);
@ -146,5 +143,4 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
static final XContentBuilderString NODE = new XContentBuilderString("node");
static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node");
}
}

132
core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
View File

@ -21,37 +21,30 @@ package org.elasticsearch.action.admin.indices.stats;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardNotFoundException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
*/
public class TransportIndicesStatsAction extends TransportBroadcastAction<IndicesStatsRequest, IndicesStatsResponse, TransportIndicesStatsAction.IndexShardStatsRequest, ShardStats> {
public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<IndicesStatsRequest, IndicesStatsResponse, ShardStats> {
private final IndicesService indicesService;
@ -60,7 +53,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
TransportService transportService, IndicesService indicesService,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, IndicesStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
IndicesStatsRequest.class, IndexShardStatsRequest.class, ThreadPool.Names.MANAGEMENT);
IndicesStatsRequest.class, ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
}
@ -68,8 +61,8 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
* Status goes across *all* shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, IndicesStatsRequest request, String[] concreteIndices) {
return clusterState.routingTable().allAssignedShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, IndicesStatsRequest request, String[] concreteIndices) {
return clusterState.routingTable().allShards(concreteIndices);
}
@Override
@ -82,45 +75,27 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices);
}
@Override
protected IndicesStatsResponse newResponse(IndicesStatsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
final List<ShardStats> shards = new ArrayList<>();
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// simply ignore non active shards
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
shards.add((ShardStats) shardResponse);
successfulShards++;
}
}
return new IndicesStatsResponse(shards.toArray(new ShardStats[shards.size()]), clusterState, shardsResponses.length(), successfulShards, failedShards, shardFailures);
protected ShardStats readShardResult(StreamInput in) throws IOException {
return ShardStats.readShardStats(in);
}
@Override
protected IndexShardStatsRequest newShardRequest(int numShards, ShardRouting shard, IndicesStatsRequest request) {
return new IndexShardStatsRequest(shard.shardId(), request);
protected IndicesStatsResponse newResponse(IndicesStatsRequest request, int totalShards, int successfulShards, int failedShards, List<ShardStats> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new IndicesStatsResponse(responses.toArray(new ShardStats[responses.size()]), totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardStats newShardResponse() {
return new ShardStats();
protected IndicesStatsRequest readRequestFrom(StreamInput in) throws IOException {
IndicesStatsRequest request = new IndicesStatsRequest();
request.readFrom(in);
return request;
}
@Override
protected ShardStats shardOperation(IndexShardStatsRequest request) {
IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(request.shardId().id());
protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting shardRouting) {
IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
// if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
if (indexShard.routingEntry() == null) {
throw new ShardNotFoundException(indexShard.shardId());
@ -128,92 +103,65 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
CommonStatsFlags flags = new CommonStatsFlags().clear();
if (request.request.docs()) {
if (request.docs()) {
flags.set(CommonStatsFlags.Flag.Docs);
}
if (request.request.store()) {
if (request.store()) {
flags.set(CommonStatsFlags.Flag.Store);
}
if (request.request.indexing()) {
if (request.indexing()) {
flags.set(CommonStatsFlags.Flag.Indexing);
flags.types(request.request.types());
flags.types(request.types());
}
if (request.request.get()) {
if (request.get()) {
flags.set(CommonStatsFlags.Flag.Get);
}
if (request.request.search()) {
if (request.search()) {
flags.set(CommonStatsFlags.Flag.Search);
flags.groups(request.request.groups());
flags.groups(request.groups());
}
if (request.request.merge()) {
if (request.merge()) {
flags.set(CommonStatsFlags.Flag.Merge);
}
if (request.request.refresh()) {
if (request.refresh()) {
flags.set(CommonStatsFlags.Flag.Refresh);
}
if (request.request.flush()) {
if (request.flush()) {
flags.set(CommonStatsFlags.Flag.Flush);
}
if (request.request.warmer()) {
if (request.warmer()) {
flags.set(CommonStatsFlags.Flag.Warmer);
}
if (request.request.queryCache()) {
if (request.queryCache()) {
flags.set(CommonStatsFlags.Flag.QueryCache);
}
if (request.request.fieldData()) {
if (request.fieldData()) {
flags.set(CommonStatsFlags.Flag.FieldData);
flags.fieldDataFields(request.request.fieldDataFields());
flags.fieldDataFields(request.fieldDataFields());
}
if (request.request.percolate()) {
if (request.percolate()) {
flags.set(CommonStatsFlags.Flag.Percolate);
}
if (request.request.segments()) {
if (request.segments()) {
flags.set(CommonStatsFlags.Flag.Segments);
}
if (request.request.completion()) {
if (request.completion()) {
flags.set(CommonStatsFlags.Flag.Completion);
flags.completionDataFields(request.request.completionFields());
flags.completionDataFields(request.completionFields());
}
if (request.request.translog()) {
if (request.translog()) {
flags.set(CommonStatsFlags.Flag.Translog);
}
if (request.request.suggest()) {
if (request.suggest()) {
flags.set(CommonStatsFlags.Flag.Suggest);
}
if (request.request.requestCache()) {
if (request.requestCache()) {
flags.set(CommonStatsFlags.Flag.RequestCache);
}
if (request.request.recovery()) {
if (request.recovery()) {
flags.set(CommonStatsFlags.Flag.Recovery);
}
return new ShardStats(indexShard, flags);
}
static class IndexShardStatsRequest extends BroadcastShardRequest {
// TODO if there are many indices, the request might hold a large indices array..., we don't really need to serialize it
IndicesStatsRequest request;
IndexShardStatsRequest() {
}
IndexShardStatsRequest(ShardId shardId, IndicesStatsRequest request) {
super(shardId, request);
this.request = request;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
request = new IndicesStatsRequest();
request.readFrom(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
request.writeTo(out);
}
return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats());
}
}

71
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java
View File

@ -22,35 +22,31 @@ package org.elasticsearch.action.admin.indices.upgrade.get;
import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.Segment;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
*
*/
public class TransportUpgradeStatusAction extends TransportBroadcastAction<UpgradeStatusRequest, UpgradeStatusResponse, TransportUpgradeStatusAction.IndexShardUpgradeStatusRequest, ShardUpgradeStatus> {
public class TransportUpgradeStatusAction extends TransportBroadcastByNodeAction<UpgradeStatusRequest, UpgradeStatusResponse, ShardUpgradeStatus> {
private final IndicesService indicesService;
@ -58,7 +54,7 @@ public class TransportUpgradeStatusAction extends TransportBroadcastAction<Upgra
public TransportUpgradeStatusAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
IndicesService indicesService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, UpgradeStatusAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
UpgradeStatusRequest.class, IndexShardUpgradeStatusRequest.class, ThreadPool.Names.MANAGEMENT);
UpgradeStatusRequest.class, ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
}
@ -66,8 +62,8 @@ public class TransportUpgradeStatusAction extends TransportBroadcastAction<Upgra
* Getting upgrade stats from *all* active shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, UpgradeStatusRequest request, String[] concreteIndices) {
return clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, UpgradeStatusRequest request, String[] concreteIndices) {
return clusterState.routingTable().allShards(concreteIndices);
}
@Override
@ -81,43 +77,26 @@ public class TransportUpgradeStatusAction extends TransportBroadcastAction<Upgra
}
@Override
protected UpgradeStatusResponse newResponse(UpgradeStatusRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
final List<ShardUpgradeStatus> shards = new ArrayList<>();
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// simply ignore non active shards
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
} else {
shards.add((ShardUpgradeStatus) shardResponse);
successfulShards++;
}
}
return new UpgradeStatusResponse(shards.toArray(new ShardUpgradeStatus[shards.size()]), shardsResponses.length(), successfulShards, failedShards, shardFailures);
protected ShardUpgradeStatus readShardResult(StreamInput in) throws IOException {
return ShardUpgradeStatus.readShardUpgradeStatus(in);
}
@Override
protected IndexShardUpgradeStatusRequest newShardRequest(int numShards, ShardRouting shard, UpgradeStatusRequest request) {
return new IndexShardUpgradeStatusRequest(shard.shardId(), request);
protected UpgradeStatusResponse newResponse(UpgradeStatusRequest request, int totalShards, int successfulShards, int failedShards, List<ShardUpgradeStatus> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new UpgradeStatusResponse(responses.toArray(new ShardUpgradeStatus[responses.size()]), totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardUpgradeStatus newShardResponse() {
return new ShardUpgradeStatus();
protected UpgradeStatusRequest readRequestFrom(StreamInput in) throws IOException {
UpgradeStatusRequest request = new UpgradeStatusRequest();
request.readFrom(in);
return request;
}
@Override
protected ShardUpgradeStatus shardOperation(IndexShardUpgradeStatusRequest request) {
IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(request.shardId().id());
protected ShardUpgradeStatus shardOperation(UpgradeStatusRequest request, ShardRouting shardRouting) {
IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
List<Segment> segments = indexShard.engine().segments(false);
long total_bytes = 0;
long to_upgrade_bytes = 0;
@ -136,16 +115,4 @@ public class TransportUpgradeStatusAction extends TransportBroadcastAction<Upgra
return new ShardUpgradeStatus(indexShard.routingEntry(), total_bytes, to_upgrade_bytes, to_upgrade_bytes_ancient);
}
static class IndexShardUpgradeStatusRequest extends BroadcastShardRequest {
IndexShardUpgradeStatusRequest() {
}
IndexShardUpgradeStatusRequest(ShardId shardId, UpgradeStatusRequest request) {
super(shardId, request);
}
}
}

4
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java
View File

@ -21,10 +21,6 @@ package org.elasticsearch.action.admin.indices.upgrade.get;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import java.io.IOException;
public class UpgradeStatusRequest extends BroadcastRequest<UpgradeStatusRequest> {

11
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
View File

@ -36,14 +36,11 @@ import java.util.Map;
import java.util.Set;
public class UpgradeStatusResponse extends BroadcastResponse implements ToXContent {
private ShardUpgradeStatus[] shards;
private Map<String, IndexUpgradeStatus> indicesUpgradeStatus;
UpgradeStatusResponse() {
}
UpgradeStatusResponse(ShardUpgradeStatus[] shards, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
@ -75,7 +72,6 @@ public class UpgradeStatusResponse extends BroadcastResponse implements ToXConte
return indicesUpgradeStats;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
@ -120,8 +116,6 @@ public class UpgradeStatusResponse extends BroadcastResponse implements ToXConte
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, getTotalBytes());
builder.byteSizeField(Fields.SIZE_TO_UPGRADE_IN_BYTES, Fields.SIZE_TO_UPGRADE, getToUpgradeBytes());
builder.byteSizeField(Fields.SIZE_TO_UPGRADE_ANCIENT_IN_BYTES, Fields.SIZE_TO_UPGRADE_ANCIENT, getToUpgradeBytesAncient());
@ -163,10 +157,8 @@ public class UpgradeStatusResponse extends BroadcastResponse implements ToXConte
}
builder.endObject();
}
builder.endObject();
}
builder.endObject();
}
return builder;
@ -186,6 +178,5 @@ public class UpgradeStatusResponse extends BroadcastResponse implements ToXConte
static final XContentBuilderString SIZE_TO_UPGRADE_ANCIENT = new XContentBuilderString("size_to_upgrade_ancient");
static final XContentBuilderString SIZE_TO_UPGRADE_IN_BYTES = new XContentBuilderString("size_to_upgrade_in_bytes");
static final XContentBuilderString SIZE_TO_UPGRADE_ANCIENT_IN_BYTES = new XContentBuilderString("size_to_upgrade_ancient_in_bytes");
}
}
}

21
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/ShardUpgradeResponse.java → core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/ShardUpgradeResult.java
View File

@ -20,9 +20,9 @@
package org.elasticsearch.action.admin.indices.upgrade.post;
import org.elasticsearch.Version;
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
@ -31,7 +31,9 @@ import java.text.ParseException;
/**
*
*/
class ShardUpgradeResponse extends BroadcastShardResponse {
class ShardUpgradeResult implements Streamable {
private ShardId shardId;
private org.apache.lucene.util.Version oldestLuceneSegment;
@ -40,16 +42,20 @@ class ShardUpgradeResponse extends BroadcastShardResponse {
private boolean primary;
ShardUpgradeResponse() {
ShardUpgradeResult() {
}
ShardUpgradeResponse(ShardId shardId, boolean primary, Version upgradeVersion, org.apache.lucene.util.Version oldestLuceneSegment) {
super(shardId);
ShardUpgradeResult(ShardId shardId, boolean primary, Version upgradeVersion, org.apache.lucene.util.Version oldestLuceneSegment) {
this.shardId = shardId;
this.primary = primary;
this.upgradeVersion = upgradeVersion;
this.oldestLuceneSegment = oldestLuceneSegment;
}
public ShardId getShardId() {
return shardId;
}
public org.apache.lucene.util.Version oldestLuceneSegment() {
return this.oldestLuceneSegment;
}
@ -65,7 +71,7 @@ class ShardUpgradeResponse extends BroadcastShardResponse {
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
shardId = ShardId.readShardId(in);
primary = in.readBoolean();
upgradeVersion = Version.readVersion(in);
try {
@ -78,10 +84,9 @@ class ShardUpgradeResponse extends BroadcastShardResponse {
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
shardId.writeTo(out);
out.writeBoolean(primary);
Version.writeVersion(upgradeVersion, out);
out.writeString(oldestLuceneSegment.toString());
}
}

105
core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
View File

@ -24,32 +24,30 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.PrimaryMissingActionException;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReferenceArray;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
@ -57,7 +55,7 @@ import static com.google.common.collect.Sets.newHashSet;
/**
* Upgrade index/indices action.
*/
public class TransportUpgradeAction extends TransportBroadcastAction<UpgradeRequest, UpgradeResponse, ShardUpgradeRequest, ShardUpgradeResponse> {
public class TransportUpgradeAction extends TransportBroadcastByNodeAction<UpgradeRequest, UpgradeResponse, ShardUpgradeResult> {
private final IndicesService indicesService;
@ -67,56 +65,40 @@ public class TransportUpgradeAction extends TransportBroadcastAction<UpgradeRequ
public TransportUpgradeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
TransportService transportService, IndicesService indicesService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, TransportUpgradeSettingsAction upgradeSettingsAction) {
super(settings, UpgradeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
UpgradeRequest.class, ShardUpgradeRequest.class, ThreadPool.Names.OPTIMIZE);
super(settings, UpgradeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, UpgradeRequest.class, ThreadPool.Names.OPTIMIZE);
this.indicesService = indicesService;
this.upgradeSettingsAction = upgradeSettingsAction;
}
@Override
protected UpgradeResponse newResponse(UpgradeRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
int successfulShards = 0;
int failedShards = 0;
List<ShardOperationFailedException> shardFailures = null;
protected UpgradeResponse newResponse(UpgradeRequest request, int totalShards, int successfulShards, int failedShards, List<ShardUpgradeResult> shardUpgradeResults, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
Map<String, Integer> successfulPrimaryShards = newHashMap();
Map<String, Tuple<Version, org.apache.lucene.util.Version>> versions = newHashMap();
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
// a non active shard, ignore...
} else if (shardResponse instanceof BroadcastShardOperationFailedException) {
failedShards++;
if (shardFailures == null) {
shardFailures = new ArrayList<>();
}
shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
for (ShardUpgradeResult result : shardUpgradeResults) {
successfulShards++;
String index = result.getShardId().getIndex();
if (result.primary()) {
Integer count = successfulPrimaryShards.get(index);
successfulPrimaryShards.put(index, count == null ? 1 : count + 1);
}
Tuple<Version, org.apache.lucene.util.Version> versionTuple = versions.get(index);
if (versionTuple == null) {
versions.put(index, new Tuple<>(result.upgradeVersion(), result.oldestLuceneSegment()));
} else {
successfulShards++;
ShardUpgradeResponse shardUpgradeResponse = (ShardUpgradeResponse) shardResponse;
String index = shardUpgradeResponse.getIndex();
if (shardUpgradeResponse.primary()) {
Integer count = successfulPrimaryShards.get(index);
successfulPrimaryShards.put(index, count == null ? 1 : count + 1);
// We already have versions for this index - let's see if we need to update them based on the current shard
Version version = versionTuple.v1();
org.apache.lucene.util.Version luceneVersion = versionTuple.v2();
// For the metadata we are interested in the _latest_ Elasticsearch version that was processing the metadata
// Since we rewrite the mapping during upgrade the metadata is always rewritten by the latest version
if (result.upgradeVersion().after(versionTuple.v1())) {
version = result.upgradeVersion();
}
Tuple<Version, org.apache.lucene.util.Version> versionTuple = versions.get(index);
if (versionTuple == null) {
versions.put(index, new Tuple<>(shardUpgradeResponse.upgradeVersion(), shardUpgradeResponse.oldestLuceneSegment()));
} else {
// We already have versions for this index - let's see if we need to update them based on the current shard
Version version = versionTuple.v1();
org.apache.lucene.util.Version luceneVersion = versionTuple.v2();
// For the metadata we are interested in the _latest_ elasticsearch version that was processing the metadata
// Since we rewrite the mapping during upgrade the metadata is always rewritten by the latest version
if (shardUpgradeResponse.upgradeVersion().after(versionTuple.v1())) {
version = shardUpgradeResponse.upgradeVersion();
}
// For the lucene version we are interested in the _oldest_ lucene version since it determines the
// oldest version that we need to support
if (shardUpgradeResponse.oldestLuceneSegment().onOrAfter(versionTuple.v2()) == false) {
luceneVersion = shardUpgradeResponse.oldestLuceneSegment();
}
versions.put(index, new Tuple<>(version, luceneVersion));
// For the lucene version we are interested in the _oldest_ lucene version since it determines the
// oldest version that we need to support
if (result.oldestLuceneSegment().onOrAfter(versionTuple.v2()) == false) {
luceneVersion = result.oldestLuceneSegment();
}
versions.put(index, new Tuple<>(version, luceneVersion));
}
}
Map<String, Tuple<org.elasticsearch.Version, String>> updatedVersions = newHashMap();
@ -133,33 +115,37 @@ public class TransportUpgradeAction extends TransportBroadcastAction<UpgradeRequ
}
}
return new UpgradeResponse(updatedVersions, shardsResponses.length(), successfulShards, failedShards, shardFailures);
return new UpgradeResponse(updatedVersions, totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected ShardUpgradeRequest newShardRequest(int numShards, ShardRouting shard, UpgradeRequest request) {
return new ShardUpgradeRequest(shard.shardId(), request);
protected ShardUpgradeResult shardOperation(UpgradeRequest request, ShardRouting shardRouting) {
IndexShard indexShard = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()).shardSafe(shardRouting.shardId().id());
org.apache.lucene.util.Version oldestLuceneSegment = indexShard.upgrade(request);
// We are using the current version of Elasticsearch as upgrade version since we update mapping to match the current version
return new ShardUpgradeResult(shardRouting.shardId(), indexShard.routingEntry().primary(), Version.CURRENT, oldestLuceneSegment);
}
@Override
protected ShardUpgradeResponse newShardResponse() {
return new ShardUpgradeResponse();
protected ShardUpgradeResult readShardResult(StreamInput in) throws IOException {
ShardUpgradeResult result = new ShardUpgradeResult();
result.readFrom(in);
return result;
}
@Override
protected ShardUpgradeResponse shardOperation(ShardUpgradeRequest request) {
IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).shardSafe(request.shardId().id());
org.apache.lucene.util.Version oldestLuceneSegment = indexShard.upgrade(request.upgradeRequest());
// We are using the current version of elasticsearch as upgrade version since we update mapping to match the current version
return new ShardUpgradeResponse(request.shardId(), indexShard.routingEntry().primary(), Version.CURRENT, oldestLuceneSegment);
protected UpgradeRequest readRequestFrom(StreamInput in) throws IOException {
UpgradeRequest request = new UpgradeRequest();
request.readFrom(in);
return request;
}
/**
* The upgrade request works against *all* shards.
*/
@Override
protected GroupShardsIterator shards(ClusterState clusterState, UpgradeRequest request, String[] concreteIndices) {
GroupShardsIterator iterator = clusterState.routingTable().allActiveShardsGrouped(concreteIndices, true);
protected ShardsIterator shards(ClusterState clusterState, UpgradeRequest request, String[] concreteIndices) {
ShardsIterator iterator = clusterState.routingTable().allShards(concreteIndices);
Set<String> indicesWithMissingPrimaries = indicesWithMissingPrimaries(clusterState, concreteIndices);
if (indicesWithMissingPrimaries.isEmpty()) {
return iterator;
@ -231,5 +217,4 @@ public class TransportUpgradeAction extends TransportBroadcastAction<UpgradeRequ
}
});
}
}

565
core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java Normal file
View File

@ -0,0 +1,565 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.support.broadcast.node;
import com.google.common.collect.Maps;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.FailedNodeException;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.NoShardAvailableActionException;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.TransportActions;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.BaseTransportResponseHandler;
import org.elasticsearch.transport.NodeShouldNotConnectException;
import org.elasticsearch.transport.TransportChannel;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Abstraction for transporting aggregated shard-level operations in a single request (NodeRequest) per-node
* and executing the shard-level operations serially on the receiving node. Each shard-level operation can produce a
* result (ShardOperationResult), these per-node shard-level results are aggregated into a single result
* (BroadcastByNodeResponse) to the coordinating node. These per-node results are aggregated into a single result (Result)
* to the client.
*
* @param <Request> the underlying client request
* @param <Response> the response to the client request
* @param <ShardOperationResult> per-shard operation results
*/
public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRequest,
Response extends BroadcastResponse,
ShardOperationResult extends Streamable> extends HandledTransportAction<Request, Response> {
private final ClusterService clusterService;
private final TransportService transportService;
final String transportNodeBroadcastAction;
public TransportBroadcastByNodeAction(
Settings settings,
String actionName,
ThreadPool threadPool,
ClusterService clusterService,
TransportService transportService,
ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
Class<Request> request,
String executor) {
super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
this.clusterService = clusterService;
this.transportService = transportService;
transportNodeBroadcastAction = actionName + "[n]";
transportService.registerRequestHandler(transportNodeBroadcastAction, new Callable<NodeRequest>() {
@Override
public NodeRequest call() throws Exception {
return new NodeRequest();
}
}, executor, new BroadcastByNodeTransportRequestHandler());
}
private final Response newResponse(
Request request,
AtomicReferenceArray responses,
List<NoShardAvailableActionException> unavailableShardExceptions,
Map<String, List<ShardRouting>> nodes,
ClusterState clusterState) {
int totalShards = 0;
int successfulShards = 0;
List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
List<ShardOperationFailedException> exceptions = new ArrayList<>();
for (int i = 0; i < responses.length(); i++) {
if (responses.get(i) instanceof FailedNodeException) {
FailedNodeException exception = (FailedNodeException) responses.get(i);
totalShards += nodes.get(exception.nodeId()).size();
for (ShardRouting shard : nodes.get(exception.nodeId())) {
exceptions.add(new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
}
} else {
NodeResponse response = (NodeResponse) responses.get(i);
broadcastByNodeResponses.addAll(response.results);
totalShards += response.getTotalShards();
successfulShards += response.getSuccessfulShards();
for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
if (!TransportActions.isShardNotAvailableException(throwable)) {
exceptions.add(new DefaultShardOperationFailedException(throwable.getIndex(), throwable.getShardId().getId(), throwable));
}
}
}
}
totalShards += unavailableShardExceptions.size();
int failedShards = exceptions.size();
return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
/**
* Deserialize a shard-level result from an input stream
*
* @param in input stream
* @return a deserialized shard-level result
* @throws IOException
*/
protected abstract ShardOperationResult readShardResult(StreamInput in) throws IOException;
/**
* Creates a new response to the underlying request.
*
* @param request the underlying request
* @param totalShards the total number of shards considered for execution of the operation
* @param successfulShards the total number of shards for which execution of the operation was successful
* @param failedShards the total number of shards for which execution of the operation failed
* @param results the per-node aggregated shard-level results
* @param shardFailures the exceptions corresponding to shard operationa failures
* @param clusterState the cluster state
* @return the response
*/
protected abstract Response newResponse(Request request, int totalShards, int successfulShards, int failedShards, List<ShardOperationResult> results, List<ShardOperationFailedException> shardFailures, ClusterState clusterState);
/**
* Deserialize a request from an input stream
*
* @param in input stream
* @return a de-serialized request
* @throws IOException
*/
protected abstract Request readRequestFrom(StreamInput in) throws IOException;
/**
* Executes the shard-level operation. This method is called once per shard serially on the receiving node.
*
* @param request the node-level request
* @param shardRouting the shard on which to execute the operation
* @return the result of the shard-level operation for the shard
*/
protected abstract ShardOperationResult shardOperation(Request request, ShardRouting shardRouting);
/**
* Determines the shards on which this operation will be executed on. The operation is executed once per shard.
*
* @param clusterState the cluster state
* @param request the underlying request
* @param concreteIndices the concrete indices on which to execute the operation
* @return the shards on which to execute the operation
*/
protected abstract ShardsIterator shards(ClusterState clusterState, Request request, String[] concreteIndices);
/**
* Executes a global block check before polling the cluster state.
*
* @param state the cluster state
* @param request the underlying request
* @return a non-null exception if the operation is blocked
*/
protected abstract ClusterBlockException checkGlobalBlock(ClusterState state, Request request);
/**
* Executes a global request-level check before polling the cluster state.
*
* @param state the cluster state
* @param request the underlying request
* @param concreteIndices the concrete indices on which to execute the operation
* @return a non-null exception if the operation if blocked
*/
protected abstract ClusterBlockException checkRequestBlock(ClusterState state, Request request, String[] concreteIndices);
@Override
protected void doExecute(Request request, ActionListener<Response> listener) {
new AsyncAction(request, listener).start();
}
protected class AsyncAction {
private final Request request;
private final ActionListener<Response> listener;
private final ClusterState clusterState;
private final DiscoveryNodes nodes;
private final Map<String, List<ShardRouting>> nodeIds;
private final AtomicReferenceArray<Object> responses;
private final AtomicInteger counter = new AtomicInteger();
private List<NoShardAvailableActionException> unavailableShardExceptions = new ArrayList<>();
protected AsyncAction(Request request, ActionListener<Response> listener) {
this.request = request;
this.listener = listener;
clusterState = clusterService.state();
nodes = clusterState.nodes();
ClusterBlockException globalBlockException = checkGlobalBlock(clusterState, request);
if (globalBlockException != null) {
throw globalBlockException;
}
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
ClusterBlockException requestBlockException = checkRequestBlock(clusterState, request, concreteIndices);
if (requestBlockException != null) {
throw requestBlockException;
}
logger.trace("resolving shards for [{}] based on cluster state version [{}]", actionName, clusterState.version());
ShardsIterator shardIt = shards(clusterState, request, concreteIndices);
nodeIds = Maps.newHashMap();
for (ShardRouting shard : shardIt.asUnordered()) {
if (shard.assignedToNode()) {
String nodeId = shard.currentNodeId();
if (!nodeIds.containsKey(nodeId)) {
nodeIds.put(nodeId, new ArrayList<ShardRouting>());
}
nodeIds.get(nodeId).add(shard);
} else {
unavailableShardExceptions.add(
new NoShardAvailableActionException(
shard.shardId(),
" no shards available for shard " + shard.toString() + " while executing " + actionName
)
);
}
}
responses = new AtomicReferenceArray<>(nodeIds.size());
}
public void start() {
if (nodeIds.size() == 0) {
try {
onCompletion();
} catch (Throwable e) {
listener.onFailure(e);
}
} else {
int nodeIndex = -1;
for (Map.Entry<String, List<ShardRouting>> entry : nodeIds.entrySet()) {
nodeIndex++;
DiscoveryNode node = nodes.get(entry.getKey());
sendNodeRequest(node, entry.getValue(), nodeIndex);
}
}
}
private void sendNodeRequest(final DiscoveryNode node, List<ShardRouting> shards, final int nodeIndex) {
try {
NodeRequest nodeRequest = new NodeRequest(node.getId(), request, shards);
transportService.sendRequest(node, transportNodeBroadcastAction, nodeRequest, new BaseTransportResponseHandler<NodeResponse>() {
@Override
public NodeResponse newInstance() {
return new NodeResponse();
}
@Override
public void handleResponse(NodeResponse response) {
onNodeResponse(node, nodeIndex, response);
}
@Override
public void handleException(TransportException exp) {
onNodeFailure(node, nodeIndex, exp);
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
});
} catch (Throwable e) {
onNodeFailure(node, nodeIndex, e);
}
}
protected void onNodeResponse(DiscoveryNode node, int nodeIndex, NodeResponse response) {
logger.trace("received response for [{}] from node [{}]", actionName, node.id());
// this is defensive to protect against the possibility of double invocation
// the current implementation of TransportService#sendRequest guards against this
// but concurrency is hard, safety is important, and the small performance loss here does not matter
if (responses.compareAndSet(nodeIndex, null, response)) {
if (counter.incrementAndGet() == responses.length()) {
onCompletion();
}
}
}
protected void onNodeFailure(DiscoveryNode node, int nodeIndex, Throwable t) {
String nodeId = node.id();
if (logger.isDebugEnabled() && !(t instanceof NodeShouldNotConnectException)) {
logger.debug("failed to execute [{}] on node [{}]", t, actionName, nodeId);
}
// this is defensive to protect against the possibility of double invocation
// the current implementation of TransportService#sendRequest guards against this
// but concurrency is hard, safety is important, and the small performance loss here does not matter
if (responses.compareAndSet(nodeIndex, null, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t))) {
if (counter.incrementAndGet() == responses.length()) {
onCompletion();
}
}
}
protected void onCompletion() {
Response response = null;
try {
response = newResponse(request, responses, unavailableShardExceptions, nodeIds, clusterState);
} catch (Throwable t) {
logger.debug("failed to combine responses from nodes", t);
listener.onFailure(t);
}
if (response != null) {
try {
listener.onResponse(response);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
}
class BroadcastByNodeTransportRequestHandler implements TransportRequestHandler<NodeRequest> {
@Override
public void messageReceived(final NodeRequest request, TransportChannel channel) throws Exception {
List<ShardRouting> shards = request.getShards();
final int totalShards = shards.size();
logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards);
final Object[] shardResultOrExceptions = new Object[totalShards];
int shardIndex = -1;
for (final ShardRouting shardRouting : shards) {
shardIndex++;
onShardOperation(request, shardResultOrExceptions, shardIndex, shardRouting);
}
List<BroadcastShardOperationFailedException> accumulatedExceptions = new ArrayList<>();
List<ShardOperationResult> results = new ArrayList<>();
for (int i = 0; i < totalShards; i++) {
if (shardResultOrExceptions[i] instanceof BroadcastShardOperationFailedException) {
accumulatedExceptions.add((BroadcastShardOperationFailedException) shardResultOrExceptions[i]);
} else {
results.add((ShardOperationResult) shardResultOrExceptions[i]);
}
}
channel.sendResponse(new NodeResponse(request.getNodeId(), totalShards, results, accumulatedExceptions));
}
private void onShardOperation(final NodeRequest request, final Object[] shardResults, final int shardIndex, final ShardRouting shardRouting) {
try {
logger.trace("[{}] executing operation for shard [{}]", actionName, shardRouting.shortSummary());
ShardOperationResult result = shardOperation(request.indicesLevelRequest, shardRouting);
shardResults[shardIndex] = result;
logger.trace("[{}] completed operation for shard [{}]", actionName, shardRouting.shortSummary());
} catch (Throwable t) {
BroadcastShardOperationFailedException e = new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", t);
e.setIndex(shardRouting.getIndex());
e.setShard(shardRouting.shardId());
shardResults[shardIndex] = e;
logger.debug("[{}] failed to execute operation for shard [{}]", e, actionName, shardRouting.shortSummary());
}
}
}
protected class NodeRequest extends TransportRequest implements IndicesRequest {
private String nodeId;
private List<ShardRouting> shards;
protected Request indicesLevelRequest;
protected NodeRequest() {
}
public NodeRequest(String nodeId, Request request, List<ShardRouting> shards) {
super(request);
this.indicesLevelRequest = request;
this.shards = shards;
this.nodeId = nodeId;
}
public List<ShardRouting> getShards() {
return shards;
}
public String getNodeId() {
return nodeId;
}
public String[] indices() {
return indicesLevelRequest.indices();
}
public IndicesOptions indicesOptions() {
return indicesLevelRequest.indicesOptions();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
indicesLevelRequest = readRequestFrom(in);
int size = in.readVInt();
shards = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
shards.add(ShardRouting.readShardRoutingEntry(in));
}
nodeId = in.readString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
indicesLevelRequest.writeTo(out);
int size = shards.size();
out.writeVInt(size);
for (int i = 0; i < size; i++) {
shards.get(i).writeTo(out);
}
out.writeString(nodeId);
}
}
class NodeResponse extends TransportResponse {
protected String nodeId;
protected int totalShards;
protected List<BroadcastShardOperationFailedException> exceptions;
protected List<ShardOperationResult> results;
public NodeResponse() {
}
public NodeResponse(String nodeId,
int totalShards,
List<ShardOperationResult> results,
List<BroadcastShardOperationFailedException> exceptions) {
this.nodeId = nodeId;
this.totalShards = totalShards;
this.results = results;
this.exceptions = exceptions;
}
public String getNodeId() {
return nodeId;
}
public int getTotalShards() {
return totalShards;
}
public int getSuccessfulShards() {
return results.size();
}
public List<BroadcastShardOperationFailedException> getExceptions() {
return exceptions;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
nodeId = in.readString();
totalShards = in.readVInt();
int resultsSize = in.readVInt();
results = new ArrayList<>(resultsSize);
for (; resultsSize > 0; resultsSize--) {
final ShardOperationResult result = in.readBoolean() ? readShardResult(in) : null;
results.add(result);
}
if (in.readBoolean()) {
int failureShards = in.readVInt();
exceptions = new ArrayList<>(failureShards);
for (int i = 0; i < failureShards; i++) {
exceptions.add(new BroadcastShardOperationFailedException(in));
}
} else {
exceptions = null;
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(nodeId);
out.writeVInt(totalShards);
out.writeVInt(results.size());
for (ShardOperationResult result : results) {
out.writeOptionalStreamable(result);
}
out.writeBoolean(exceptions != null);
if (exceptions != null) {
int failureShards = exceptions.size();
out.writeVInt(failureShards);
for (int i = 0; i < failureShards; i++) {
exceptions.get(i).writeTo(out);
}
}
}
}
/**
* Can be used for implementations of {@link #shardOperation(BroadcastRequest, ShardRouting) shardOperation} for
* which there is no shard-level return value.
*/
public final static class EmptyResult implements Streamable {
public static EmptyResult INSTANCE = new EmptyResult();
private EmptyResult() {
}
@Override
public void readFrom(StreamInput in) throws IOException {
}
@Override
public void writeTo(StreamOutput out) throws IOException {
}
public static EmptyResult readEmptyResultFrom(StreamInput in) {
return INSTANCE;
}
}
}

2
core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
View File

@ -97,7 +97,7 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
listener.onFailure(blockException);
return;
}
logger.trace("can't execute due to a cluster block: [{}], retrying", blockException);
logger.trace("can't execute due to a cluster block, retrying", blockException);
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override

13
core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
View File

@ -46,6 +46,7 @@ import org.elasticsearch.env.EnvironmentModule;
import org.elasticsearch.indices.breaker.CircuitBreakerModule;
import org.elasticsearch.monitor.MonitorService;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.PluginsModule;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.search.SearchModule;
@ -55,6 +56,7 @@ import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.netty.NettyTransport;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@ -82,6 +84,7 @@ public class TransportClient extends AbstractClient {
public static class Builder {
private Settings settings = Settings.EMPTY;
private List<Class<? extends Plugin>> pluginClasses = new ArrayList<>();
private boolean loadConfigSettings = true;
/**
@ -108,6 +111,14 @@ public class TransportClient extends AbstractClient {
return this;
}
/**
* Add the given plugin to the client when it is created.
*/
public Builder addPlugin(Class<? extends Plugin> pluginClass) {
pluginClasses.add(pluginClass);
return this;
}
/**
* Builds a new instance of the transport client.
*/
@ -122,7 +133,7 @@ public class TransportClient extends AbstractClient {
.build();
Environment environment = tuple.v2();
PluginsService pluginsService = new PluginsService(settings, tuple.v2());
PluginsService pluginsService = new PluginsService(settings, tuple.v2(), pluginClasses);
this.settings = pluginsService.updatedSettings();
Version version = Version.CURRENT;

51
core/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
View File

@ -32,28 +32,63 @@ import java.util.Map;
*/
public class ClusterInfo {
private final Map<String, DiskUsage> usages;
private final Map<String, DiskUsage> leastAvailableSpaceUsage;
private final Map<String, DiskUsage> mostAvailableSpaceUsage;
final Map<String, Long> shardSizes;
public static final ClusterInfo EMPTY = new ClusterInfo();
private final Map<ShardRouting, String> routingToDataPath;
private ClusterInfo() {
this.usages = Collections.emptyMap();
this.shardSizes = Collections.emptyMap();
protected ClusterInfo() {
this(Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
}
public ClusterInfo(Map<String, DiskUsage> usages, Map<String, Long> shardSizes) {
this.usages = usages;
/**
* Creates a new ClusterInfo instance.
*
* @param leastAvailableSpaceUsage a node id to disk usage mapping for the path that has the least available space on the node.
* @param mostAvailableSpaceUsage a node id to disk usage mapping for the path that has the most available space on the node.
* @param shardSizes a shardkey to size in bytes mapping per shard.
* @param routingToDataPath the shard routing to datapath mapping
* @see #shardIdentifierFromRouting
*/
public ClusterInfo(final Map<String, DiskUsage> leastAvailableSpaceUsage, final Map<String, DiskUsage> mostAvailableSpaceUsage, final Map<String, Long> shardSizes, Map<ShardRouting, String> routingToDataPath) {
this.leastAvailableSpaceUsage = leastAvailableSpaceUsage;
this.shardSizes = shardSizes;
this.mostAvailableSpaceUsage = mostAvailableSpaceUsage;
this.routingToDataPath = routingToDataPath;
}
public Map<String, DiskUsage> getNodeDiskUsages() {
return this.usages;
/**
* Returns a node id to disk usage mapping for the path that has the least available space on the node.
*/
public Map<String, DiskUsage> getNodeLeastAvailableDiskUsages() {
return this.leastAvailableSpaceUsage;
}
/**
* Returns a node id to disk usage mapping for the path that has the most available space on the node.
*/
public Map<String, DiskUsage> getNodeMostAvailableDiskUsages() {
return this.mostAvailableSpaceUsage;
}
/**
* Returns the shard size for the given shard routing or <code>null</code> it that metric is not available.
*/
public Long getShardSize(ShardRouting shardRouting) {
return shardSizes.get(shardIdentifierFromRouting(shardRouting));
}
/**
* Returns the nodes absolute data-path the given shard is allocated on or <code>null</code> if the information is not available.
*/
public String getDataPath(ShardRouting shardRouting) {
return routingToDataPath.get(shardRouting);
}
/**
* Returns the shard size for the given shard routing or <code>defaultValue</code> it that metric is not available.
*/
public long getShardSize(ShardRouting shardRouting, long defaultValue) {
Long shardSize = getShardSize(shardRouting);
return shardSize == null ? defaultValue : shardSize;

10
core/src/main/java/org/elasticsearch/cluster/DiskUsage.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
public class DiskUsage {
final String nodeId;
final String nodeName;
final String path;
final long totalBytes;
final long freeBytes;
@ -35,11 +36,12 @@ public class DiskUsage {
* Create a new DiskUsage, if {@code totalBytes} is 0, {@get getFreeDiskAsPercentage}
* will always return 100.0% free
*/
public DiskUsage(String nodeId, String nodeName, long totalBytes, long freeBytes) {
public DiskUsage(String nodeId, String nodeName, String path, long totalBytes, long freeBytes) {
this.nodeId = nodeId;
this.nodeName = nodeName;
this.freeBytes = freeBytes;
this.totalBytes = totalBytes;
this.path = path;
}
public String getNodeId() {
@ -50,6 +52,10 @@ public class DiskUsage {
return nodeName;
}
public String getPath() {
return path;
}
public double getFreeDiskAsPercentage() {
// We return 100.0% in order to fail "open", in that if we have invalid
// numbers for the total bytes, it's as if we don't know disk usage.
@ -77,7 +83,7 @@ public class DiskUsage {
@Override
public String toString() {
return "[" + nodeId + "][" + nodeName + "] free: " + new ByteSizeValue(getFreeBytes()) +
return "[" + nodeId + "][" + nodeName + "][" + path + "] free: " + new ByteSizeValue(getFreeBytes()) +
"[" + Strings.format1Decimals(getFreeDiskAsPercentage(), "%") + "]";
}
}

112
core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
@ -32,12 +31,13 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.node.settings.NodeSettingsService;
@ -47,7 +47,6 @@ import org.elasticsearch.transport.ReceiveTimeoutTransportException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
/**
* InternalClusterInfoService provides the ClusterInfoService interface,
@ -67,7 +66,9 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
private volatile TimeValue updateFrequency;
private volatile Map<String, DiskUsage> usages;
private volatile Map<String, DiskUsage> leastAvailableSpaceUsages;
private volatile Map<String, DiskUsage> mostAvailableSpaceUsages;
private volatile Map<ShardRouting, String> shardRoutingToDataPath;
private volatile Map<String, Long> shardSizes;
private volatile boolean isMaster = false;
private volatile boolean enabled;
@ -84,7 +85,9 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
TransportIndicesStatsAction transportIndicesStatsAction, ClusterService clusterService,
ThreadPool threadPool) {
super(settings);
this.usages = Collections.emptyMap();
this.leastAvailableSpaceUsages = Collections.emptyMap();
this.mostAvailableSpaceUsages = Collections.emptyMap();
this.shardRoutingToDataPath = Collections.emptyMap();
this.shardSizes = Collections.emptyMap();
this.transportNodesStatsAction = transportNodesStatsAction;
this.transportIndicesStatsAction = transportIndicesStatsAction;
@ -200,9 +203,16 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
if (logger.isTraceEnabled()) {
logger.trace("Removing node from cluster info: {}", removedNode.getId());
}
Map<String, DiskUsage> newUsages = new HashMap<>(usages);
newUsages.remove(removedNode.getId());
usages = Collections.unmodifiableMap(newUsages);
if (leastAvailableSpaceUsages.containsKey(removedNode.getId())) {
Map<String, DiskUsage> newMaxUsages = new HashMap<>(leastAvailableSpaceUsages);
newMaxUsages.remove(removedNode.getId());
leastAvailableSpaceUsages = Collections.unmodifiableMap(newMaxUsages);
}
if (mostAvailableSpaceUsages.containsKey(removedNode.getId())) {
Map<String, DiskUsage> newMinUsages = new HashMap<>(mostAvailableSpaceUsages);
newMinUsages.remove(removedNode.getId());
mostAvailableSpaceUsages = Collections.unmodifiableMap(newMinUsages);
}
}
}
}
@ -210,7 +220,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
@Override
public ClusterInfo getClusterInfo() {
return new ClusterInfo(usages, shardSizes);
return new ClusterInfo(leastAvailableSpaceUsages, mostAvailableSpaceUsages, shardSizes, shardRoutingToDataPath);
}
@Override
@ -313,27 +323,11 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
CountDownLatch nodeLatch = updateNodeStats(new ActionListener<NodesStatsResponse>() {
@Override
public void onResponse(NodesStatsResponse nodeStatses) {
Map<String, DiskUsage> newUsages = new HashMap<>();
for (NodeStats nodeStats : nodeStatses.getNodes()) {
if (nodeStats.getFs() == null) {
logger.warn("Unable to retrieve node FS stats for {}", nodeStats.getNode().name());
} else {
long available = 0;
long total = 0;
for (FsInfo.Path info : nodeStats.getFs()) {
available += info.getAvailable().bytes();
total += info.getTotal().bytes();
}
String nodeId = nodeStats.getNode().id();
String nodeName = nodeStats.getNode().getName();
if (logger.isTraceEnabled()) {
logger.trace("node: [{}], total disk: {}, available disk: {}", nodeId, total, available);
}
newUsages.put(nodeId, new DiskUsage(nodeId, nodeName, total, available));
}
}
usages = Collections.unmodifiableMap(newUsages);
Map<String, DiskUsage> newLeastAvaiableUsages = new HashMap<>();
Map<String, DiskUsage> newMostAvaiableUsages = new HashMap<>();
fillDiskUsagePerNode(logger, nodeStatses.getNodes(), newLeastAvaiableUsages, newMostAvaiableUsages);
leastAvailableSpaceUsages = Collections.unmodifiableMap(newLeastAvaiableUsages);
mostAvailableSpaceUsages = Collections.unmodifiableMap(newMostAvaiableUsages);
}
@Override
@ -349,7 +343,8 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
logger.warn("Failed to execute NodeStatsAction for ClusterInfoUpdateJob", e);
}
// we empty the usages list, to be safe - we don't know what's going on.
usages = Collections.emptyMap();
leastAvailableSpaceUsages = Collections.emptyMap();
mostAvailableSpaceUsages = Collections.emptyMap();
}
}
});
@ -358,16 +353,11 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
@Override
public void onResponse(IndicesStatsResponse indicesStatsResponse) {
ShardStats[] stats = indicesStatsResponse.getShards();
HashMap<String, Long> newShardSizes = new HashMap<>();
for (ShardStats s : stats) {
long size = s.getStats().getStore().sizeInBytes();
String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
if (logger.isTraceEnabled()) {
logger.trace("shard: {} size: {}", sid, size);
}
newShardSizes.put(sid, size);
}
final HashMap<String, Long> newShardSizes = new HashMap<>();
final HashMap<ShardRouting, String> newShardRoutingToDataPath = new HashMap<>();
buildShardLevelInfo(logger, stats, newShardSizes, newShardRoutingToDataPath);
shardSizes = Collections.unmodifiableMap(newShardSizes);
shardRoutingToDataPath = Collections.unmodifiableMap(newShardRoutingToDataPath);
}
@Override
@ -384,6 +374,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
}
// we empty the usages list, to be safe - we don't know what's going on.
shardSizes = Collections.emptyMap();
shardRoutingToDataPath = Collections.emptyMap();
}
}
});
@ -412,5 +403,46 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
}
}
static void buildShardLevelInfo(ESLogger logger, ShardStats[] stats, HashMap<String, Long> newShardSizes, HashMap<ShardRouting, String> newShardRoutingToDataPath) {
for (ShardStats s : stats) {
newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
long size = s.getStats().getStore().sizeInBytes();
String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
if (logger.isTraceEnabled()) {
logger.trace("shard: {} size: {}", sid, size);
}
newShardSizes.put(sid, size);
}
}
static void fillDiskUsagePerNode(ESLogger logger, NodeStats[] nodeStatsArray, Map<String, DiskUsage> newLeastAvaiableUsages, Map<String, DiskUsage> newMostAvaiableUsages) {
for (NodeStats nodeStats : nodeStatsArray) {
if (nodeStats.getFs() == null) {
logger.warn("Unable to retrieve node FS stats for {}", nodeStats.getNode().name());
} else {
FsInfo.Path leastAvailablePath = null;
FsInfo.Path mostAvailablePath = null;
for (FsInfo.Path info : nodeStats.getFs()) {
if (leastAvailablePath == null) {
assert mostAvailablePath == null;
mostAvailablePath = leastAvailablePath = info;
} else if (leastAvailablePath.getAvailable().bytes() > info.getAvailable().bytes()){
leastAvailablePath = info;
} else if (mostAvailablePath.getAvailable().bytes() < info.getAvailable().bytes()) {
mostAvailablePath = info;
}
}
String nodeId = nodeStats.getNode().id();
String nodeName = nodeStats.getNode().getName();
if (logger.isTraceEnabled()) {
logger.trace("node: [{}], most available: total disk: {}, available disk: {} / least available: total disk: {}, available disk: {}", nodeId, mostAvailablePath.getTotal(), leastAvailablePath.getAvailable(), leastAvailablePath.getTotal(), leastAvailablePath.getAvailable());
}
newLeastAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, leastAvailablePath.getPath(), leastAvailablePath.getTotal().bytes(), leastAvailablePath.getAvailable().bytes()));
newMostAvaiableUsages.put(nodeId, new DiskUsage(nodeId, nodeName, mostAvailablePath.getPath(), mostAvailablePath.getTotal().bytes(), mostAvailablePath.getAvailable().bytes()));
}
}
}
}

33
core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.cluster.routing;
import com.carrotsearch.hppc.IntSet;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.UnmodifiableIterator;
@ -223,6 +224,38 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
return new GroupShardsIterator(set);
}
public ShardsIterator allShards(String[] indices) {
return allShardsSatisfyingPredicate(indices, Predicates.<ShardRouting>alwaysTrue(), false);
}
public ShardsIterator allShardsIncludingRelocationTargets(String[] indices) {
return allShardsSatisfyingPredicate(indices, Predicates.<ShardRouting>alwaysTrue(), true);
}
// TODO: replace with JDK 8 native java.util.function.Predicate
private ShardsIterator allShardsSatisfyingPredicate(String[] indices, Predicate<ShardRouting> predicate, boolean includeRelocationTargets) {
// use list here since we need to maintain identity across shards
List<ShardRouting> shards = new ArrayList<>();
for (String index : indices) {
IndexRoutingTable indexRoutingTable = index(index);
if (indexRoutingTable == null) {
continue;
// we simply ignore indices that don't exists (make sense for operations that use it currently)
}
for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
for (ShardRouting shardRouting : indexShardRoutingTable) {
if (predicate.apply(shardRouting)) {
shards.add(shardRouting);
if (includeRelocationTargets && shardRouting.relocating()) {
shards.add(shardRouting.buildTargetRelocatingShard());
}
}
}
}
}
return new PlainShardsIterator(shards);
}
/**
* All the *active* primary shards for the provided indices grouped (each group is a single element, consisting
* of the primary shard). This is handy for components that expect to get group iterators, but still want in some

51
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
View File

@ -164,7 +164,7 @@ public class DiskThresholdDecider extends AllocationDecider {
@Override
public void onNewInfo(ClusterInfo info) {
Map<String, DiskUsage> usages = info.getNodeDiskUsages();
Map<String, DiskUsage> usages = info.getNodeLeastAvailableDiskUsages();
if (usages != null) {
boolean reroute = false;
String explanation = "";
@ -313,13 +313,16 @@ public class DiskThresholdDecider extends AllocationDecider {
* If subtractShardsMovingAway is set then the size of shards moving away is subtracted from the total size
* of all shards
*/
public static long sizeOfRelocatingShards(RoutingNode node, ClusterInfo clusterInfo, boolean subtractShardsMovingAway) {
public static long sizeOfRelocatingShards(RoutingNode node, ClusterInfo clusterInfo, boolean subtractShardsMovingAway, String dataPath) {
long totalSize = 0;
for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING)) {
if (routing.initializing() && routing.relocatingNodeId() != null) {
totalSize += getShardSize(routing, clusterInfo);
} else if (subtractShardsMovingAway && routing.relocating()) {
totalSize -= getShardSize(routing, clusterInfo);
String actualPath = clusterInfo.getDataPath(routing);
if (dataPath.equals(actualPath)) {
if (routing.initializing() && routing.relocatingNodeId() != null) {
totalSize += getShardSize(routing, clusterInfo);
} else if (subtractShardsMovingAway && routing.relocating()) {
totalSize -= getShardSize(routing, clusterInfo);
}
}
}
return totalSize;
@ -339,7 +342,9 @@ public class DiskThresholdDecider extends AllocationDecider {
final double usedDiskThresholdLow = 100.0 - DiskThresholdDecider.this.freeDiskThresholdLow;
final double usedDiskThresholdHigh = 100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh;
DiskUsage usage = getDiskUsage(node, allocation);
ClusterInfo clusterInfo = allocation.clusterInfo();
Map<String, DiskUsage> usages = clusterInfo.getNodeMostAvailableDiskUsages();
DiskUsage usage = getDiskUsage(node, allocation, usages);
// First, check that the node currently over the low watermark
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
// Cache the used disk percentage for displaying disk percentages consistent with documentation
@ -441,17 +446,26 @@ public class DiskThresholdDecider extends AllocationDecider {
@Override
public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
if (shardRouting.currentNodeId().equals(node.nodeId()) == false) {
throw new IllegalArgumentException("Shard [" + shardRouting + "] is not allocated on node: [" + node.nodeId() + "]");
}
final Decision decision = earlyTerminate(allocation);
if (decision != null) {
return decision;
}
DiskUsage usage = getDiskUsage(node, allocation);
final ClusterInfo clusterInfo = allocation.clusterInfo();
final Map<String, DiskUsage> usages = clusterInfo.getNodeLeastAvailableDiskUsages();
final DiskUsage usage = getDiskUsage(node, allocation, usages);
final String dataPath = clusterInfo.getDataPath(shardRouting);
// If this node is already above the high threshold, the shard cannot remain (get it off!)
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
long freeBytes = usage.getFreeBytes();
final double freeDiskPercentage = usage.getFreeDiskAsPercentage();
final long freeBytes = usage.getFreeBytes();
if (logger.isDebugEnabled()) {
logger.debug("node [{}] has {}% free disk ({} bytes)", node.nodeId(), freeDiskPercentage, freeBytes);
}
if (dataPath == null || usage.getPath().equals(dataPath) == false) {
return allocation.decision(Decision.YES, NAME, "shard is not allocated on the most utilized disk");
}
if (freeBytes < freeBytesThresholdHigh.bytes()) {
if (logger.isDebugEnabled()) {
logger.debug("less than the required {} free bytes threshold ({} bytes free) on node {}, shard cannot remain",
@ -472,9 +486,8 @@ public class DiskThresholdDecider extends AllocationDecider {
return allocation.decision(Decision.YES, NAME, "enough disk for shard to remain on node, free: [%s]", new ByteSizeValue(freeBytes));
}
private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation) {
private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation, Map<String, DiskUsage> usages) {
ClusterInfo clusterInfo = allocation.clusterInfo();
Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
DiskUsage usage = usages.get(node.nodeId());
if (usage == null) {
// If there is no usage, and we have other nodes in the cluster,
@ -487,8 +500,8 @@ public class DiskThresholdDecider extends AllocationDecider {
}
if (includeRelocations) {
long relocatingShardsSize = sizeOfRelocatingShards(node, clusterInfo, true);
DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().name(),
long relocatingShardsSize = sizeOfRelocatingShards(node, clusterInfo, true, usage.getPath());
DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().name(), usage.getPath(),
usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize);
if (logger.isTraceEnabled()) {
logger.trace("usage without relocations: {}", usage);
@ -508,7 +521,7 @@ public class DiskThresholdDecider extends AllocationDecider {
*/
public DiskUsage averageUsage(RoutingNode node, Map<String, DiskUsage> usages) {
if (usages.size() == 0) {
return new DiskUsage(node.nodeId(), node.node().name(), 0, 0);
return new DiskUsage(node.nodeId(), node.node().name(), "_na_", 0, 0);
}
long totalBytes = 0;
long freeBytes = 0;
@ -516,7 +529,7 @@ public class DiskThresholdDecider extends AllocationDecider {
totalBytes += du.getTotalBytes();
freeBytes += du.getFreeBytes();
}
return new DiskUsage(node.nodeId(), node.node().name(), totalBytes / usages.size(), freeBytes / usages.size());
return new DiskUsage(node.nodeId(), node.node().name(), "_na_", totalBytes / usages.size(), freeBytes / usages.size());
}
/**
@ -528,8 +541,8 @@ public class DiskThresholdDecider extends AllocationDecider {
*/
public double freeDiskPercentageAfterShardAssigned(DiskUsage usage, Long shardSize) {
shardSize = (shardSize == null) ? 0 : shardSize;
DiskUsage newUsage = new DiskUsage(usage.getNodeId(), usage.getNodeName(),
usage.getTotalBytes(), usage.getFreeBytes() - shardSize);
DiskUsage newUsage = new DiskUsage(usage.getNodeId(), usage.getNodeName(), usage.getPath(),
usage.getTotalBytes(), usage.getFreeBytes() - shardSize);
return newUsage.getFreeDiskAsPercentage();
}
@ -600,7 +613,7 @@ public class DiskThresholdDecider extends AllocationDecider {
return allocation.decision(Decision.YES, NAME, "cluster info unavailable");
}
final Map<String, DiskUsage> usages = clusterInfo.getNodeDiskUsages();
final Map<String, DiskUsage> usages = clusterInfo.getNodeLeastAvailableDiskUsages();
// Fail open if there are no disk usages available
if (usages.isEmpty()) {
if (logger.isTraceEnabled()) {

3
core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java
View File

@ -46,8 +46,7 @@ public class LZFCompressor implements Compressor {
public LZFCompressor() {
this.decoder = ChunkDecoderFactory.safeInstance();
Loggers.getLogger(LZFCompressor.class).debug("using encoder [{}] and decoder[{}] ",
this.decoder.getClass().getSimpleName());
Loggers.getLogger(LZFCompressor.class).debug("using decoder[{}] ", this.decoder.getClass().getSimpleName());
}
@Override

14
core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.*;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.ByteArray;
import java.io.*;
import java.net.HttpURLConnection;
@ -37,9 +36,7 @@ import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
/**
*
@ -152,12 +149,6 @@ public class HttpDownloadHelper {
} catch (FileNotFoundException | NoSuchFileException e) {
// checksum file doesn't exist
return false;
} catch (IOException e) {
if (ExceptionsHelper.unwrapCause(e) instanceof FileNotFoundException) {
// checksum file didn't exist
return false;
}
throw e;
} finally {
IOUtils.deleteFilesIgnoringExceptions(checksumFile);
}
@ -378,9 +369,6 @@ public class HttpDownloadHelper {
responseCode == HttpURLConnection.HTTP_MOVED_TEMP ||
responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
String newLocation = httpConnection.getHeaderField("Location");
String message = aSource
+ (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
: "") + " moved to " + newLocation;
URL newURL = new URL(newLocation);
if (!redirectionAllowed(aSource, newURL)) {
return null;
@ -426,7 +414,7 @@ public class HttpDownloadHelper {
}
}
if (is == null) {
throw new IOException("Can't get " + source + " to " + dest, lastEx);
throw lastEx;
}
os = Files.newOutputStream(dest);

20
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -56,8 +57,10 @@ import org.elasticsearch.plugins.PluginsService;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -315,8 +318,23 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
throw t;
}
}
if (path == null) {
path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), this);
// TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
// that's being relocated/replicated we know how large it will become once it's done copying:
// Count up how many shards are currently on each data path:
Map<Path,Integer> dataPathToShardCount = new HashMap<>();
for(IndexShard shard : this) {
Path dataPath = shard.shardPath().getRootStatePath();
Integer curCount = dataPathToShardCount.get(dataPath);
if (curCount == null) {
curCount = 0;
}
dataPathToShardCount.put(dataPath, curCount+1);
}
path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(),
dataPathToShardCount);
logger.debug("{} creating using a new path [{}]", shardId, path);
} else {
logger.debug("{} creating using an existing path [{}]", shardId, path);

30
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -104,8 +104,9 @@ class DocumentParser implements Closeable {
if (token != XContentParser.Token.START_OBJECT) {
throw new MapperParsingException("Malformed content, must start with an object");
}
boolean emptyDoc = false;
if (mapping.root.isEnabled()) {
boolean emptyDoc = false;
token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
@ -113,23 +114,24 @@ class DocumentParser implements Closeable {
} else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist");
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.preParse(context);
}
if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.postParse(context);
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.preParse(context);
}
} else {
if (mapping.root.isEnabled() == false) {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.postParse(context);
}
// try to parse the next token, this should be null if the object is ended properly

27
core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
View File

@ -199,19 +199,27 @@ public final class ShardPath {
}
public static ShardPath selectNewPathForShard(NodeEnvironment env, ShardId shardId, @IndexSettings Settings indexSettings,
long avgShardSizeInBytes, Iterable<IndexShard> shards) throws IOException {
long avgShardSizeInBytes, Map<Path,Integer> dataPathToShardCount) throws IOException {
final Path dataPath;
final Path statePath;
final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
if (NodeEnvironment.hasCustomDataPath(indexSettings)) {
dataPath = env.resolveCustomLocation(indexSettings, shardId);
statePath = env.nodePaths()[0].resolve(shardId);
} else {
Map<Path,Long> estReservedBytes = getEstimatedReservedBytes(env, avgShardSizeInBytes, shards);
long totFreeSpace = 0;
for (NodeEnvironment.NodePath nodePath : env.nodePaths()) {
totFreeSpace += nodePath.fileStore.getUsableSpace();
}
// TODO: this is a hack!! We should instead keep track of incoming (relocated) shards since we know
// how large they will be once they're done copying, instead of a silly guess for such cases:
// Very rough heurisic of how much disk space we expect the shard will use over its lifetime, the max of current average
// shard size across the cluster and 5% of the total available free space on this node:
long estShardSizeInBytes = Math.max(avgShardSizeInBytes, (long) (totFreeSpace/20.0));
// TODO - do we need something more extensible? Yet, this does the job for now...
final NodeEnvironment.NodePath[] paths = env.nodePaths();
@ -220,10 +228,11 @@ public final class ShardPath {
for (NodeEnvironment.NodePath nodePath : paths) {
FileStore fileStore = nodePath.fileStore;
long usableBytes = fileStore.getUsableSpace();
Long reservedBytes = estReservedBytes.get(nodePath.path);
if (reservedBytes != null) {
// Deduct estimated reserved bytes from usable space:
usableBytes -= reservedBytes;
// Deduct estimated reserved bytes from usable space:
Integer count = dataPathToShardCount.get(nodePath.path);
if (count != null) {
usableBytes -= estShardSizeInBytes * count;
}
if (usableBytes > maxUsableBytes) {
maxUsableBytes = usableBytes;
@ -235,6 +244,8 @@ public final class ShardPath {
dataPath = statePath;
}
final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId);
}

2
core/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -254,7 +254,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
if (indexShard.routingEntry() == null) {
continue;
}
IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) });
IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats()) });
if (!statsByShard.containsKey(indexService.index())) {
statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
} else {

31
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -73,6 +73,7 @@ import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.percolator.PercolatorModule;
import org.elasticsearch.percolator.PercolatorService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.PluginsModule;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.repositories.RepositoriesModule;
@ -95,6 +96,8 @@ import org.elasticsearch.watcher.ResourceWatcherService;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@ -117,19 +120,22 @@ public class Node implements Releasable {
private final PluginsService pluginsService;
private final Client client;
public Node() {
this(Settings.Builder.EMPTY_SETTINGS, true);
/**
* Constructs a node with the given settings.
*
* @param preparedSettings Base settings to configure the node with
* @param loadConfigSettings true if settings should also be loaded and merged from configuration files
*/
public Node(Settings preparedSettings, boolean loadConfigSettings) {
this(preparedSettings, loadConfigSettings, Version.CURRENT, Collections.<Class<? extends Plugin>>emptyList());
}
public Node(Settings preparedSettings, boolean loadConfigSettings) {
Node(Settings preparedSettings, boolean loadConfigSettings, Version version, Collection<Class<? extends Plugin>> classpathPlugins) {
final Settings pSettings = settingsBuilder().put(preparedSettings)
.put(Client.CLIENT_TYPE_SETTING, CLIENT_TYPE).build();
Tuple<Settings, Environment> tuple = InternalSettingsPreparer.prepareSettings(pSettings, loadConfigSettings);
tuple = new Tuple<>(TribeService.processSettings(tuple.v1()), tuple.v2());
// The only place we can actually fake the version a node is running on:
Version version = pSettings.getAsVersion("tests.mock.version", Version.CURRENT);
ESLogger logger = Loggers.getLogger(Node.class, tuple.v1().get("name"));
logger.info("version[{}], pid[{}], build[{}/{}]", version, JvmInfo.jvmInfo().pid(), Build.CURRENT.hashShort(), Build.CURRENT.timestamp());
@ -141,7 +147,7 @@ public class Node implements Releasable {
env.configFile(), Arrays.toString(env.dataFiles()), env.logsFile(), env.pluginsFile());
}
this.pluginsService = new PluginsService(tuple.v1(), tuple.v2());
this.pluginsService = new PluginsService(tuple.v1(), tuple.v2(), classpathPlugins);
this.settings = pluginsService.updatedSettings();
// create the environment based on the finalized (processed) view of the settings
this.environment = new Environment(this.settings());
@ -421,15 +427,4 @@ public class Node implements Releasable {
public Injector injector() {
return this.injector;
}
public static void main(String[] args) throws Exception {
final Node node = new Node();
node.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
node.close();
}
});
}
}

10
core/src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -463,17 +463,15 @@ public class PluginManager {
if (version != null) {
// Elasticsearch new download service uses groupId org.elasticsearch.plugin from 2.0.0
if (user == null) {
// TODO Update to https
if (!Strings.isNullOrEmpty(System.getProperty(PROPERTY_SUPPORT_STAGING_URLS))) {
addUrl(urls, String.format(Locale.ROOT, "http://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", version, Build.CURRENT.hashShort(), name, version, name, version));
addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", version, Build.CURRENT.hashShort(), name, version, name, version));
}
addUrl(urls, String.format(Locale.ROOT, "http://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%s/%s/%s-%s.zip", name, version, name, version));
addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%s/%s/%s-%s.zip", name, version, name, version));
} else {
// Elasticsearch old download service
// TODO Update to https
addUrl(urls, String.format(Locale.ROOT, "http://download.elastic.co/%1$s/%2$s/%2$s-%3$s.zip", user, name, version));
addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/%1$s/%2$s/%2$s-%3$s.zip", user, name, version));
// Maven central repository
addUrl(urls, String.format(Locale.ROOT, "http://search.maven.org/remotecontent?filepath=%1$s/%2$s/%3$s/%2$s-%3$s.zip", user.replace('.', '/'), name, version));
addUrl(urls, String.format(Locale.ROOT, "https://search.maven.org/remotecontent?filepath=%1$s/%2$s/%3$s/%2$s-%3$s.zip", user.replace('.', '/'), name, version));
// Sonatype repository
addUrl(urls, String.format(Locale.ROOT, "https://oss.sonatype.org/service/local/repositories/releases/content/%1$s/%2$s/%3$s/%2$s-%3$s.zip", user.replace('.', '/'), name, version));
// Github repository

51
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -70,10 +70,10 @@ public class PluginsService extends AbstractComponent {
/**
* We keep around a list of plugins
*/
private final ImmutableList<Tuple<PluginInfo, Plugin>> plugins;
private final List<Tuple<PluginInfo, Plugin>> plugins;
private final PluginsInfo info;
private final ImmutableMap<Plugin, List<OnModuleReference>> onModuleReferences;
private final Map<Plugin, List<OnModuleReference>> onModuleReferences;
static class OnModuleReference {
public final Class<? extends Module> moduleClass;
@ -89,20 +89,19 @@ public class PluginsService extends AbstractComponent {
* Constructs a new PluginService
* @param settings The settings of the system
* @param environment The environment of the system
* @param classpathPlugins Plugins that exist in the classpath which should be loaded
*/
public PluginsService(Settings settings, Environment environment) {
public PluginsService(Settings settings, Environment environment, Collection<Class<? extends Plugin>> classpathPlugins) {
super(settings);
ImmutableList.Builder<Tuple<PluginInfo, Plugin>> tupleBuilder = ImmutableList.builder();
List<Tuple<PluginInfo, Plugin>> tupleBuilder = new ArrayList<>();
// first we load specified plugins via 'plugin.types' settings parameter.
// this is a hack for what is between unit and integration tests...
String[] defaultPluginsClasses = settings.getAsArray("plugin.types");
for (String pluginClass : defaultPluginsClasses) {
Plugin plugin = loadPlugin(pluginClass, settings, getClass().getClassLoader());
PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), false, "NA", true, pluginClass, false);
// first we load plugins that are on the classpath. this is for tests and transport clients
for (Class<? extends Plugin> pluginClass : classpathPlugins) {
Plugin plugin = loadPlugin(pluginClass, settings);
PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), false, "NA", true, pluginClass.getName(), false);
if (logger.isTraceEnabled()) {
logger.trace("plugin loaded from settings [{}]", pluginInfo);
logger.trace("plugin loaded from classpath [{}]", pluginInfo);
}
tupleBuilder.add(new Tuple<>(pluginInfo, plugin));
}
@ -115,7 +114,7 @@ public class PluginsService extends AbstractComponent {
throw new IllegalStateException("Unable to initialize plugins", ex);
}
plugins = tupleBuilder.build();
plugins = Collections.unmodifiableList(tupleBuilder);
info = new PluginsInfo();
for (Tuple<PluginInfo, Plugin> tuple : plugins) {
info.add(tuple.v1());
@ -128,7 +127,7 @@ public class PluginsService extends AbstractComponent {
for (Tuple<PluginInfo, Plugin> tuple : plugins) {
PluginInfo info = tuple.v1();
if (info.isJvm()) {
jvmPlugins.put(tuple.v2().name(), tuple.v2());
jvmPlugins.put(info.getName(), tuple.v2());
}
if (info.isSite()) {
sitePlugins.add(info.getName());
@ -151,7 +150,7 @@ public class PluginsService extends AbstractComponent {
logger.info("loaded {}, sites {}", jvmPlugins.keySet(), sitePlugins);
MapBuilder<Plugin, List<OnModuleReference>> onModuleReferences = MapBuilder.newMapBuilder();
Map<Plugin, List<OnModuleReference>> onModuleReferences = new HashMap<>();
for (Plugin plugin : jvmPlugins.values()) {
List<OnModuleReference> list = new ArrayList<>();
for (Method method : plugin.getClass().getMethods()) {
@ -173,10 +172,10 @@ public class PluginsService extends AbstractComponent {
onModuleReferences.put(plugin, list);
}
}
this.onModuleReferences = onModuleReferences.immutableMap();
this.onModuleReferences = Collections.unmodifiableMap(onModuleReferences);
}
public ImmutableList<Tuple<PluginInfo, Plugin>> plugins() {
public List<Tuple<PluginInfo, Plugin>> plugins() {
return plugins;
}
@ -355,7 +354,8 @@ public class PluginsService extends AbstractComponent {
if (pluginInfo.isJvm()) {
// reload lucene SPI with any new services from the plugin
reloadLuceneSPI(loader);
plugin = loadPlugin(pluginInfo.getClassname(), settings, loader);
Class<? extends Plugin> pluginClass = loadPluginClass(pluginInfo.getClassname(), loader);
plugin = loadPlugin(pluginClass, settings);
} else {
plugin = new SitePlugin(pluginInfo.getName(), pluginInfo.getDescription());
}
@ -384,10 +384,16 @@ public class PluginsService extends AbstractComponent {
TokenizerFactory.reloadTokenizers(loader);
}
private Plugin loadPlugin(String className, Settings settings, ClassLoader loader) {
private Class<? extends Plugin> loadPluginClass(String className, ClassLoader loader) {
try {
Class<? extends Plugin> pluginClass = loader.loadClass(className).asSubclass(Plugin.class);
return loader.loadClass(className).asSubclass(Plugin.class);
} catch (ClassNotFoundException e) {
throw new ElasticsearchException("Could not find plugin class [" + className + "]", e);
}
}
private Plugin loadPlugin(Class<? extends Plugin> pluginClass, Settings settings) {
try {
try {
return pluginClass.getConstructor(Settings.class).newInstance(settings);
} catch (NoSuchMethodException e) {
@ -395,13 +401,12 @@ public class PluginsService extends AbstractComponent {
return pluginClass.getConstructor().newInstance();
} catch (NoSuchMethodException e1) {
throw new ElasticsearchException("No constructor for [" + pluginClass + "]. A plugin class must " +
"have either an empty default constructor or a single argument constructor accepting a " +
"Settings instance");
"have either an empty default constructor or a single argument constructor accepting a " +
"Settings instance");
}
}
} catch (Throwable e) {
throw new ElasticsearchException("Failed to load plugin class [" + className + "]", e);
throw new ElasticsearchException("Failed to load plugin class [" + pluginClass.getName() + "]", e);
}
}
}

21
core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.cat;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Strings;
@ -116,19 +115,19 @@ public class RestRecoveryAction extends AbstractCatAction {
Table t = getTableWithHeader(request);
for (String index : response.shardResponses().keySet()) {
for (String index : response.shardRecoveryStates().keySet()) {
List<ShardRecoveryResponse> shardRecoveryResponses = response.shardResponses().get(index);
if (shardRecoveryResponses.size() == 0) {
List<RecoveryState> shardRecoveryStates = response.shardRecoveryStates().get(index);
if (shardRecoveryStates.size() == 0) {
continue;
}
// Sort ascending by shard id for readability
CollectionUtil.introSort(shardRecoveryResponses, new Comparator<ShardRecoveryResponse>() {
CollectionUtil.introSort(shardRecoveryStates, new Comparator<RecoveryState>() {
@Override
public int compare(ShardRecoveryResponse o1, ShardRecoveryResponse o2) {
int id1 = o1.recoveryState().getShardId().id();
int id2 = o2.recoveryState().getShardId().id();
public int compare(RecoveryState o1, RecoveryState o2) {
int id1 = o1.getShardId().id();
int id2 = o2.getShardId().id();
if (id1 < id2) {
return -1;
} else if (id1 > id2) {
@ -139,12 +138,10 @@ public class RestRecoveryAction extends AbstractCatAction {
}
});
for (ShardRecoveryResponse shardResponse : shardRecoveryResponses) {
RecoveryState state = shardResponse.recoveryState();
for (RecoveryState state: shardRecoveryStates) {
t.startRow();
t.addCell(index);
t.addCell(shardResponse.getShardId());
t.addCell(state.getShardId().id());
t.addCell(state.getTimer().time());
t.addCell(state.getType().toString().toLowerCase(Locale.ROOT));
t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT));

15
core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
View File

@ -21,7 +21,11 @@ package org.elasticsearch.rest.action.cat;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.segments.*;
import org.elasticsearch.action.admin.indices.segments.IndexSegments;
import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
import org.elasticsearch.action.admin.indices.segments.ShardSegments;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.Strings;
@ -29,7 +33,10 @@ import org.elasticsearch.common.Table;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.engine.Segment;
import org.elasticsearch.rest.*;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.action.support.RestActionListener;
import org.elasticsearch.rest.action.support.RestResponseListener;
import org.elasticsearch.rest.action.support.RestTable;
@ -120,8 +127,8 @@ public class RestSegmentsAction extends AbstractCatAction {
for (Segment segment : segments) {
table.startRow();
table.addCell(shardSegment.getIndex());
table.addCell(shardSegment.getShardId());
table.addCell(shardSegment.getShardRouting().getIndex());
table.addCell(shardSegment.getShardRouting().getId());
table.addCell(shardSegment.getShardRouting().primary() ? "p" : "r");
table.addCell(nodes.get(shardSegment.getShardRouting().currentNodeId()).getHostAddress());
table.addCell(shardSegment.getShardRouting().currentNodeId());

2
core/src/main/java/org/elasticsearch/rest/action/support/RestTable.java
View File

@ -68,7 +68,7 @@ public class RestTable {
public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException {
RestRequest request = channel.request();
boolean verbose = request.paramAsBoolean("v", true);
boolean verbose = request.paramAsBoolean("v", false);
List<DisplayHeader> headers = buildDisplayHeaders(table, request);
int[] width = buildWidths(table, request, verbose, headers);

34
core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.transport;
import java.lang.reflect.Constructor;
import java.util.concurrent.Callable;
/**
*
@ -28,20 +29,19 @@ import java.lang.reflect.Constructor;
public class RequestHandlerRegistry<Request extends TransportRequest> {
private final String action;
private final Constructor<Request> requestConstructor;
private final TransportRequestHandler<Request> handler;
private final boolean forceExecution;
private final String executor;
private final Callable<Request> requestFactory;
RequestHandlerRegistry(String action, Class<Request> request, TransportRequestHandler<Request> handler,
String executor, boolean forceExecution) {
this(action, new ReflectionFactory<>(request), handler, executor, forceExecution);
}
public RequestHandlerRegistry(String action, Callable<Request> requestFactory, TransportRequestHandler<Request> handler, String executor, boolean forceExecution) {
this.action = action;
try {
this.requestConstructor = request.getDeclaredConstructor();
} catch (NoSuchMethodException e) {
throw new IllegalStateException("failed to create constructor (does it have a default constructor?) for request " + request, e);
}
this.requestConstructor.setAccessible(true);
this.requestFactory = requestFactory;
assert newRequest() != null;
this.handler = handler;
this.forceExecution = forceExecution;
@ -54,7 +54,7 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
public Request newRequest() {
try {
return requestConstructor.newInstance();
return requestFactory.call();
} catch (Exception e) {
throw new IllegalStateException("failed to instantiate request ", e);
}
@ -71,4 +71,22 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
public String getExecutor() {
return executor;
}
private final static class ReflectionFactory<Request> implements Callable<Request> {
private final Constructor<Request> requestConstructor;
public ReflectionFactory(Class<Request> request) {
try {
this.requestConstructor = request.getDeclaredConstructor();
} catch (NoSuchMethodException e) {
throw new IllegalStateException("failed to create constructor (does it have a default constructor?) for request " + request, e);
}
this.requestConstructor.setAccessible(true);
}
@Override
public Request call() throws Exception {
return requestConstructor.newInstance();
}
}
}

2
core/src/main/java/org/elasticsearch/transport/TransportModule.java
View File

@ -85,7 +85,7 @@ public class TransportModule extends AbstractModule {
bind(TransportService.class).asEagerSingleton();
} else {
if (transportServices.containsKey(typeName) == false) {
throw new IllegalArgumentException("Unknown TransportService [" + typeName + "]");
throw new IllegalArgumentException("Unknown TransportService type [" + typeName + "], known types are: " + transportServices.keySet());
}
bind(TransportService.class).to(transportServices.get(typeName)).asEagerSingleton();
}

33
core/src/main/java/org/elasticsearch/transport/TransportService.java
View File

@ -22,8 +22,6 @@ package org.elasticsearch.transport;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.settings.ClusterDynamicSettings;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
@ -35,12 +33,21 @@ import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.concurrent.*;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
@ -399,6 +406,18 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
registerRequestHandler(action, request, executor, false, handler);
}
/**
* Registers a new request handler
* @param action The action the request handler is associated with
* @param requestFactory a callable to be used construct new instances for streaming
* @param executor The executor the request handling will be executed on
* @param handler The handler itself that implements the request handling
*/
public <Request extends TransportRequest> void registerRequestHandler(String action, Callable<Request> requestFactory, String executor, TransportRequestHandler<Request> handler) {
RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(action, requestFactory, handler, executor, false);
registerRequestHandler(reg);
}
/**
* Registers a new request handler
* @param action The action the request handler is associated with
@ -408,8 +427,12 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
* @param handler The handler itself that implements the request handling
*/
public <Request extends TransportRequest> void registerRequestHandler(String action, Class<Request> request, String executor, boolean forceExecution, TransportRequestHandler<Request> handler) {
RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(action, request, handler, executor, forceExecution);
registerRequestHandler(reg);
}
protected <Request extends TransportRequest> void registerRequestHandler(RequestHandlerRegistry<Request> reg) {
synchronized (requestHandlerMutex) {
RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(action, request, handler, executor, forceExecution);
RequestHandlerRegistry replaced = requestHandlers.get(reg.getAction());
requestHandlers = MapBuilder.newMapBuilder(requestHandlers).put(reg.getAction(), reg).immutableMap();
if (replaced != null) {

36
core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
View File

@ -95,6 +95,7 @@ import org.elasticsearch.search.action.SearchServiceTransportAction;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportChannel;
@ -107,12 +108,14 @@ import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@ -141,11 +144,15 @@ public class IndicesRequestIT extends ESIntegTestCase {
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.extendArray("plugin.types", InterceptingTransportService.TestPlugin.class.getName())
.build();
protected Settings nodeSettings(int ordinal) {
// must set this independently of the plugin so it overrides MockTransportService
return Settings.builder().put(super.nodeSettings(ordinal))
.put(TransportModule.TRANSPORT_SERVICE_TYPE_KEY, "intercepting").build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(InterceptingTransportService.TestPlugin.class);
}
@Before
@ -395,7 +402,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
@Test
public void testOptimize() {
String optimizeShardAction = OptimizeAction.NAME + "[s]";
String optimizeShardAction = OptimizeAction.NAME + "[n]";
interceptTransportActions(optimizeShardAction);
OptimizeRequest optimizeRequest = new OptimizeRequest(randomIndicesOrAliases());
@ -419,7 +426,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
@Test
public void testClearCache() {
String clearCacheAction = ClearIndicesCacheAction.NAME + "[s]";
String clearCacheAction = ClearIndicesCacheAction.NAME + "[n]";
interceptTransportActions(clearCacheAction);
ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(randomIndicesOrAliases());
@ -431,7 +438,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
@Test
public void testRecovery() {
String recoveryAction = RecoveryAction.NAME + "[s]";
String recoveryAction = RecoveryAction.NAME + "[n]";
interceptTransportActions(recoveryAction);
RecoveryRequest recoveryRequest = new RecoveryRequest(randomIndicesOrAliases());
@ -443,7 +450,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
@Test
public void testSegments() {
String segmentsAction = IndicesSegmentsAction.NAME + "[s]";
String segmentsAction = IndicesSegmentsAction.NAME + "[n]";
interceptTransportActions(segmentsAction);
IndicesSegmentsRequest segmentsRequest = new IndicesSegmentsRequest(randomIndicesOrAliases());
@ -455,7 +462,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
@Test
public void testIndicesStats() {
String indicesStats = IndicesStatsAction.NAME + "[s]";
String indicesStats = IndicesStatsAction.NAME + "[n]";
interceptTransportActions(indicesStats);
IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest().indices(randomIndicesOrAliases());
@ -856,10 +863,6 @@ public class IndicesRequestIT extends ESIntegTestCase {
public void onModule(TransportModule transportModule) {
transportModule.addTransportService("intercepting", InterceptingTransportService.class);
}
@Override
public Settings additionalSettings() {
return Settings.builder().put(TransportModule.TRANSPORT_SERVICE_TYPE_KEY, "intercepting").build();
}
}
private final Set<String> actions = new HashSet<>();
@ -888,6 +891,11 @@ public class IndicesRequestIT extends ESIntegTestCase {
super.registerRequestHandler(action, request, executor, forceExecution, new InterceptingRequestHandler(action, handler));
}
@Override
public <Request extends TransportRequest> void registerRequestHandler(String action, Callable<Request> requestFactory, String executor, TransportRequestHandler<Request> handler) {
super.registerRequestHandler(action, requestFactory, executor, new InterceptingRequestHandler(action, handler));
}
private class InterceptingRequestHandler implements TransportRequestHandler {
private final TransportRequestHandler requestHandler;

422
core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java Normal file
View File

@ -0,0 +1,422 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.support.broadcast.node;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilter;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.DummyTransportAddress;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.cluster.TestClusterService;
import org.elasticsearch.test.transport.CapturingTransport;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportChannel;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportResponseOptions;
import org.elasticsearch.transport.TransportService;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.object.HasToString.hasToString;
public class TransportBroadcastByNodeActionTests extends ESTestCase {
private static final String TEST_INDEX = "test-index";
private static final String TEST_CLUSTER = "test-cluster";
private static ThreadPool THREAD_POOL;
private TestClusterService clusterService;
private CapturingTransport transport;
private TransportService transportService;
private TestTransportBroadcastByNodeAction action;
public static class Request extends BroadcastRequest<Request> {
public Request() {
}
public Request(String[] indices) {
super(indices);
}
}
public static class Response extends BroadcastResponse {
public Response() {
}
public Response(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
super(totalShards, successfulShards, failedShards, shardFailures);
}
}
class TestTransportBroadcastByNodeAction extends TransportBroadcastByNodeAction<Request, Response, TransportBroadcastByNodeAction.EmptyResult> {
private final Map<ShardRouting, Object> shards = new HashMap<>();
public TestTransportBroadcastByNodeAction(Settings settings, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Class<Request> request, String executor) {
super(settings, "indices:admin/test", THREAD_POOL, TransportBroadcastByNodeActionTests.this.clusterService, transportService, actionFilters, indexNameExpressionResolver, request, executor);
}
@Override
protected EmptyResult readShardResult(StreamInput in) throws IOException {
return EmptyResult.readEmptyResultFrom(in);
}
@Override
protected Response newResponse(Request request, int totalShards, int successfulShards, int failedShards, List<EmptyResult> emptyResults, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
return new Response(totalShards, successfulShards, failedShards, shardFailures);
}
@Override
protected Request readRequestFrom(StreamInput in) throws IOException {
final Request request = new Request();
request.readFrom(in);
return request;
}
@Override
protected EmptyResult shardOperation(Request request, ShardRouting shardRouting) {
if (rarely()) {
shards.put(shardRouting, Boolean.TRUE);
return EmptyResult.INSTANCE;
} else {
ElasticsearchException e = new ElasticsearchException("operation failed");
shards.put(shardRouting, e);
throw e;
}
}
@Override
protected ShardsIterator shards(ClusterState clusterState, Request request, String[] concreteIndices) {
return clusterState.routingTable().allShards(new String[]{TEST_INDEX});
}
@Override
protected ClusterBlockException checkGlobalBlock(ClusterState state, Request request) {
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
}
@Override
protected ClusterBlockException checkRequestBlock(ClusterState state, Request request, String[] concreteIndices) {
return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices);
}
public Map<ShardRouting, Object> getResults() {
return shards;
}
}
class MyResolver extends IndexNameExpressionResolver {
public MyResolver() {
super(Settings.EMPTY);
}
@Override
public String[] concreteIndices(ClusterState state, IndicesRequest request) {
return request.indices();
}
}
@BeforeClass
public static void startThreadPool() {
THREAD_POOL = new ThreadPool(TransportBroadcastByNodeActionTests.class.getSimpleName());
}
@Before
public void setUp() throws Exception {
super.setUp();
transport = new CapturingTransport();
clusterService = new TestClusterService(THREAD_POOL);
transportService = new TransportService(transport, THREAD_POOL);
transportService.start();
setClusterState(clusterService, TEST_INDEX);
action = new TestTransportBroadcastByNodeAction(
Settings.EMPTY,
transportService,
new ActionFilters(new HashSet<ActionFilter>()),
new MyResolver(),
Request.class,
ThreadPool.Names.SAME
);
}
void setClusterState(TestClusterService clusterService, String index) {
int numberOfNodes = randomIntBetween(3, 5);
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index);
int shardIndex = -1;
for (int i = 0; i < numberOfNodes; i++) {
final DiscoveryNode node = newNode(i);
discoBuilder = discoBuilder.put(node);
int numberOfShards = randomIntBetween(0, 10);
for (int j = 0; j < numberOfShards; j++) {
final ShardId shardId = new ShardId(index, ++shardIndex);
ShardRouting shard = TestShardRouting.newShardRouting(index, shardId.getId(), node.id(), true, ShardRoutingState.STARTED, 1);
IndexShardRoutingTable.Builder indexShard = new IndexShardRoutingTable.Builder(shardId);
indexShard.addShard(shard);
indexRoutingTable.addIndexShard(indexShard.build());
}
}
discoBuilder.localNodeId(newNode(0).id());
discoBuilder.masterNodeId(newNode(numberOfNodes - 1).id());
ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName(TEST_CLUSTER));
stateBuilder.nodes(discoBuilder);
stateBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable.build()).build());
ClusterState clusterState = stateBuilder.build();
clusterService.setState(clusterState);
}
static DiscoveryNode newNode(int nodeId) {
return new DiscoveryNode("node_" + nodeId, DummyTransportAddress.INSTANCE, Version.CURRENT);
}
@AfterClass
public static void destroyThreadPool() {
ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
// since static must set to null to be eligible for collection
THREAD_POOL = null;
}
public void testGlobalBlock() {
Request request = new Request(new String[]{TEST_INDEX});
PlainActionFuture<Response> listener = new PlainActionFuture<>();
ClusterBlocks.Builder block = ClusterBlocks.builder()
.addGlobalBlock(new ClusterBlock(1, "", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
try {
action.new AsyncAction(request, listener).start();
fail("expected ClusterBlockException");
} catch (ClusterBlockException expected) {
}
}
public void testRequestBlock() {
Request request = new Request(new String[]{TEST_INDEX});
PlainActionFuture<Response> listener = new PlainActionFuture<>();
ClusterBlocks.Builder block = ClusterBlocks.builder()
.addIndexBlock(TEST_INDEX, new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
try {
action.new AsyncAction(request, listener).start();
fail("expected ClusterBlockException");
} catch (ClusterBlockException expected) {
}
}
public void testOneRequestIsSentToEachNodeHoldingAShard() {
Request request = new Request(new String[]{TEST_INDEX});
PlainActionFuture<Response> listener = new PlainActionFuture<>();
action.new AsyncAction(request, listener).start();
Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.capturedRequestsByTargetNode();
ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX});
Set<String> set = new HashSet<>();
for (ShardRouting shard : shardIt.asUnordered()) {
set.add(shard.currentNodeId());
}
// check a request was sent to the right number of nodes
assertEquals(set.size(), capturedRequests.size());
// check requests were sent to the right nodes
assertEquals(set, capturedRequests.keySet());
for (Map.Entry<String, List<CapturingTransport.CapturedRequest>> entry : capturedRequests.entrySet()) {
// check one request was sent to each node
assertEquals(1, entry.getValue().size());
}
}
public void testOperationExecution() throws Exception {
ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX});
Set<ShardRouting> shards = new HashSet<>();
String nodeId = shardIt.asUnordered().iterator().next().currentNodeId();
for (ShardRouting shard : shardIt.asUnordered()) {
if (nodeId.equals(shard.currentNodeId())) {
shards.add(shard);
}
}
final TransportBroadcastByNodeAction.BroadcastByNodeTransportRequestHandler handler =
action.new BroadcastByNodeTransportRequestHandler();
TestTransportChannel channel = new TestTransportChannel();
handler.messageReceived(action.new NodeRequest(nodeId, new Request(), new ArrayList<>(shards)), channel);
// check the operation was executed only on the expected shards
assertEquals(shards, action.getResults().keySet());
TransportResponse response = channel.getCapturedResponse();
assertTrue(response instanceof TransportBroadcastByNodeAction.NodeResponse);
TransportBroadcastByNodeAction.NodeResponse nodeResponse = (TransportBroadcastByNodeAction.NodeResponse)response;
// check the operation was executed on the correct node
assertEquals("node id", nodeId, nodeResponse.getNodeId());
int successfulShards = 0;
int failedShards = 0;
for (Object result : action.getResults().values()) {
if (!(result instanceof ElasticsearchException)) {
successfulShards++;
} else {
failedShards++;
}
}
// check the operation results
assertEquals("successful shards", successfulShards, nodeResponse.getSuccessfulShards());
assertEquals("total shards", action.getResults().size(), nodeResponse.getTotalShards());
assertEquals("failed shards", failedShards, nodeResponse.getExceptions().size());
List<BroadcastShardOperationFailedException> exceptions = nodeResponse.getExceptions();
for (BroadcastShardOperationFailedException exception : exceptions) {
assertThat(exception.getMessage(), is("operation indices:admin/test failed"));
assertThat(exception, hasToString(containsString("operation failed")));
}
}
public void testResultAggregation() throws ExecutionException, InterruptedException {
Request request = new Request(new String[]{TEST_INDEX});
PlainActionFuture<Response> listener = new PlainActionFuture<>();
action.new AsyncAction(request, listener).start();
Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.capturedRequestsByTargetNode();
transport.clear();
ShardsIterator shardIt = clusterService.state().getRoutingTable().allShards(new String[]{TEST_INDEX});
Map<String, List<ShardRouting>> map = new HashMap<>();
for (ShardRouting shard : shardIt.asUnordered()) {
if (!map.containsKey(shard.currentNodeId())) {
map.put(shard.currentNodeId(), new ArrayList<ShardRouting>());
}
map.get(shard.currentNodeId()).add(shard);
}
int totalShards = 0;
int totalSuccessfulShards = 0;
int totalFailedShards = 0;
for (Map.Entry<String, List<CapturingTransport.CapturedRequest>> entry : capturedRequests.entrySet()) {
List<BroadcastShardOperationFailedException> exceptions = new ArrayList<>();
long requestId = entry.getValue().get(0).requestId;
if (rarely()) {
// simulate node failure
totalShards += map.get(entry.getKey()).size();
totalFailedShards += map.get(entry.getKey()).size();
transport.handleResponse(requestId, new Exception());
} else {
List<ShardRouting> shards = map.get(entry.getKey());
List<TransportBroadcastByNodeAction.EmptyResult> shardResults = new ArrayList<>();
for (ShardRouting shard : shards) {
totalShards++;
if (rarely()) {
// simulate operation failure
totalFailedShards++;
exceptions.add(new BroadcastShardOperationFailedException(shard.shardId(), "operation indices:admin/test failed"));
} else {
shardResults.add(TransportBroadcastByNodeAction.EmptyResult.INSTANCE);
}
}
totalSuccessfulShards += shardResults.size();
TransportBroadcastByNodeAction.NodeResponse nodeResponse = action.new NodeResponse(entry.getKey(), shards.size(), shardResults, exceptions);
transport.handleResponse(requestId, nodeResponse);
}
}
Response response = listener.get();
assertEquals("total shards", totalShards, response.getTotalShards());
assertEquals("successful shards", totalSuccessfulShards, response.getSuccessfulShards());
assertEquals("failed shards", totalFailedShards, response.getFailedShards());
assertEquals("accumulated exceptions", totalFailedShards, response.getShardFailures().length);
}
public class TestTransportChannel implements TransportChannel {
private TransportResponse capturedResponse;
public TransportResponse getCapturedResponse() {
return capturedResponse;
}
@Override
public String action() {
return null;
}
@Override
public String getProfileName() {
return "";
}
@Override
public void sendResponse(TransportResponse response) throws IOException {
capturedResponse = response;
}
@Override
public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
}
@Override
public void sendResponse(Throwable error) throws IOException {
}
}
}

8
core/src/test/java/org/elasticsearch/benchmark/recovery/ReplicaRecoveryBenchmark.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.benchmark.recovery;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -30,6 +29,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.SizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.BackgroundIndexer;
import org.elasticsearch.transport.TransportModule;
@ -128,12 +128,12 @@ public class ReplicaRecoveryBenchmark {
long currentTime = System.currentTimeMillis();
long currentDocs = indexer.totalIndexedDocs();
RecoveryResponse recoveryResponse = client1.admin().indices().prepareRecoveries(INDEX_NAME).setActiveOnly(true).get();
List<ShardRecoveryResponse> indexRecoveries = recoveryResponse.shardResponses().get(INDEX_NAME);
List<RecoveryState> indexRecoveries = recoveryResponse.shardRecoveryStates().get(INDEX_NAME);
long translogOps;
long bytes;
if (indexRecoveries.size() > 0) {
translogOps = indexRecoveries.get(0).recoveryState().getTranslog().recoveredOperations();
bytes = recoveryResponse.shardResponses().get(INDEX_NAME).get(0).recoveryState().getIndex().recoveredBytes();
translogOps = indexRecoveries.get(0).getTranslog().recoveredOperations();
bytes = recoveryResponse.shardRecoveryStates().get(INDEX_NAME).get(0).getIndex().recoveredBytes();
} else {
bytes = lastBytes = 0;
translogOps = lastTranslogOps = 0;

15
core/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.benchmark.scripts.expression;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.Client;
@ -28,13 +29,18 @@ import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService.ScriptType;
import org.elasticsearch.search.sort.ScriptSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.joda.time.PeriodType;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Random;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@ -102,10 +108,11 @@ public class ScriptComparisonBenchmark {
static Client setupIndex() throws Exception {
// create cluster
Settings settings = settingsBuilder().put("plugin.types", NativeScriptPlugin.class.getName())
.put("name", "node1")
.build();
Node node1 = nodeBuilder().clusterName(clusterName).settings(settings).node();
Settings settings = settingsBuilder().put("name", "node1")
.put("cluster.name", clusterName).build();
Collection<Class<? extends Plugin>> plugins = Collections.<Class<? extends Plugin>>singletonList(NativeScriptPlugin.class);
Node node1 = new MockNode(settings, true, Version.CURRENT, plugins);
node1.start();
Client client = node1.client();
client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet();

12
core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsConstantScoreBenchmark.java
View File

@ -18,14 +18,19 @@
*/
package org.elasticsearch.benchmark.scripts.score;
import org.elasticsearch.Version;
import org.elasticsearch.benchmark.scripts.score.plugin.NativeScriptExamplesPlugin;
import org.elasticsearch.benchmark.scripts.score.script.NativeConstantForLoopScoreScript;
import org.elasticsearch.benchmark.scripts.score.script.NativeConstantScoreScript;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
@ -46,10 +51,13 @@ public class ScriptsConstantScoreBenchmark extends BasicScriptBenchmark {
init(maxTerms);
List<Results> allResults = new ArrayList<>();
Settings settings = settingsBuilder().put("plugin.types", NativeScriptExamplesPlugin.class.getName()).build();
String clusterName = ScriptsConstantScoreBenchmark.class.getSimpleName();
Node node1 = nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings).put("name", "node1")).node();
Settings settings = settingsBuilder().put("name", "node1")
.put("cluster.name", clusterName).build();
Collection<Class<? extends Plugin>> plugins = Collections.<Class<? extends Plugin>>singletonList(NativeScriptExamplesPlugin.class);
Node node1 = new MockNode(settings, true, Version.CURRENT, plugins);
node1.start();
Client client = node1.client();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet();

13
core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsScoreBenchmark.java
View File

@ -18,13 +18,18 @@
*/
package org.elasticsearch.benchmark.scripts.score;
import org.elasticsearch.Version;
import org.elasticsearch.benchmark.scripts.score.plugin.NativeScriptExamplesPlugin;
import org.elasticsearch.benchmark.scripts.score.script.NativeNaiveTFIDFScoreScript;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
@ -46,10 +51,12 @@ public class ScriptsScoreBenchmark extends BasicScriptBenchmark {
boolean runMVEL = false;
init(maxTerms);
List<Results> allResults = new ArrayList<>();
Settings settings = settingsBuilder().put("plugin.types", NativeScriptExamplesPlugin.class.getName()).build();
String clusterName = ScriptsScoreBenchmark.class.getSimpleName();
Node node1 = nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings).put("name", "node1")).node();
Settings settings = settingsBuilder().put("name", "node1")
.put("cluster.name", clusterName).build();
Collection<Class<? extends Plugin>> plugins = Collections.<Class<? extends Plugin>>singletonList(NativeScriptExamplesPlugin.class);
Node node1 = new MockNode(settings, true, Version.CURRENT, plugins);
node1.start();
Client client = node1.client();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet();

13
core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsScorePayloadSumBenchmark.java
View File

@ -18,14 +18,19 @@
*/
package org.elasticsearch.benchmark.scripts.score;
import org.elasticsearch.Version;
import org.elasticsearch.benchmark.scripts.score.plugin.NativeScriptExamplesPlugin;
import org.elasticsearch.benchmark.scripts.score.script.NativePayloadSumNoRecordScoreScript;
import org.elasticsearch.benchmark.scripts.score.script.NativePayloadSumScoreScript;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
@ -46,10 +51,12 @@ public class ScriptsScorePayloadSumBenchmark extends BasicScriptBenchmark {
init(maxTerms);
List<Results> allResults = new ArrayList<>();
Settings settings = settingsBuilder().put("plugin.types", NativeScriptExamplesPlugin.class.getName()).build();
String clusterName = ScriptsScoreBenchmark.class.getSimpleName();
Node node1 = nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings).put("name", "node1")).node();
Settings settings = settingsBuilder().put("name", "node1")
.put("cluster.name", clusterName).build();
Collection<Class<? extends Plugin>> plugins = Collections.<Class<? extends Plugin>>singletonList(NativeScriptExamplesPlugin.class);
Node node1 = new MockNode(settings, true, Version.CURRENT, plugins);
node1.start();
Client client = node1.client();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet();

20
core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
View File

@ -48,6 +48,7 @@ import org.elasticsearch.transport.TransportResponseHandler;
import org.elasticsearch.transport.TransportService;
import org.junit.Test;
import java.util.Collection;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ -63,12 +64,13 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTests {
@Override
protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
TransportClient client = TransportClient.builder().settings(Settings.builder()
TransportClient client = TransportClient.builder()
.settings(Settings.builder()
.put("client.transport.sniff", false)
.put("node.name", "transport_client_" + this.getTestName())
.put("plugin.types", InternalTransportService.TestPlugin.class.getName())
.put(headersSettings)
.build()).build();
.build())
.addPlugin(InternalTransportService.TestPlugin.class).build();
client.addTransportAddress(address);
return client;
@ -76,15 +78,17 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTests {
@Test
public void testWithSniffing() throws Exception {
TransportClient client = TransportClient.builder().settings(Settings.builder()
TransportClient client = TransportClient.builder()
.settings(Settings.builder()
.put("client.transport.sniff", true)
.put("cluster.name", "cluster1")
.put("node.name", "transport_client_" + this.getTestName() + "_1")
.put("client.transport.nodes_sampler_interval", "1s")
.put("plugin.types", InternalTransportService.TestPlugin.class.getName())
.put("client.transport.nodes_sampler_interval", "1s")
.put(HEADER_SETTINGS)
.put("path.home", createTempDir().toString())
.build()).build();
.put("path.home", createTempDir().toString()).build())
.addPlugin(InternalTransportService.TestPlugin.class)
.build();
try {
client.addTransportAddress(address);

61
core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
View File

@ -31,10 +31,16 @@ import org.elasticsearch.action.support.ActionFilter;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RoutingNodes;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
@ -47,6 +53,7 @@ import org.hamcrest.Matchers;
import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@ -146,16 +153,23 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
return Settings.builder()
// manual collection or upon cluster forming.
.put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT, "1s")
.putArray("plugin.types", TestPlugin.class.getName(), MockTransportService.TestPlugin.class.getName())
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(TestPlugin.class,
MockTransportService.TestPlugin.class);
}
@Test
public void testClusterInfoServiceCollectsInformation() throws Exception {
internalCluster().startNodesAsync(2,
Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, "200ms").build())
.get();
assertAcked(prepareCreate("test").setSettings(settingsBuilder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL, 0).build()));
assertAcked(prepareCreate("test").setSettings(settingsBuilder()
.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL, 0)
.put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE).build()));
ensureGreen("test");
InternalTestCluster internalTestCluster = internalCluster();
// Get the cluster info service on the master node
@ -164,13 +178,18 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
infoService.addListener(listener);
ClusterInfo info = listener.get();
assertNotNull("info should not be null", info);
Map<String, DiskUsage> usages = info.getNodeDiskUsages();
Map<String, Long> shardSizes = info.shardSizes;
assertNotNull(usages);
final Map<String, DiskUsage> leastUsages = info.getNodeLeastAvailableDiskUsages();
final Map<String, DiskUsage> mostUsages = info.getNodeMostAvailableDiskUsages();
final Map<String, Long> shardSizes = info.shardSizes;
assertNotNull(leastUsages);
assertNotNull(shardSizes);
assertThat("some usages are populated", usages.values().size(), Matchers.equalTo(2));
assertThat("some usages are populated", leastUsages.values().size(), Matchers.equalTo(2));
assertThat("some shard sizes are populated", shardSizes.values().size(), greaterThan(0));
for (DiskUsage usage : usages.values()) {
for (DiskUsage usage : leastUsages.values()) {
logger.info("--> usage: {}", usage);
assertThat("usage has be retrieved", usage.getFreeBytes(), greaterThan(0L));
}
for (DiskUsage usage : mostUsages.values()) {
logger.info("--> usage: {}", usage);
assertThat("usage has be retrieved", usage.getFreeBytes(), greaterThan(0L));
}
@ -178,6 +197,21 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
logger.info("--> shard size: {}", size);
assertThat("shard size is greater than 0", size, greaterThan(0L));
}
ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getMasterName());
ClusterState state = clusterService.state();
RoutingNodes routingNodes = state.getRoutingNodes();
for (ShardRouting shard : routingNodes.getRoutingTable().allShards()) {
String dataPath = info.getDataPath(shard);
assertNotNull(dataPath);
String nodeId = shard.currentNodeId();
DiscoveryNode discoveryNode = state.getNodes().get(nodeId);
IndicesService indicesService = internalTestCluster.getInstance(IndicesService.class, discoveryNode.getName());
IndexService indexService = indicesService.indexService(shard.index());
IndexShard indexShard = indexService.shard(shard.id());
assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath);
}
}
@Test
@ -197,14 +231,14 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
infoService.updateOnce();
ClusterInfo info = listener.get();
assertNotNull("failed to collect info", info);
assertThat("some usages are populated", info.getNodeDiskUsages().size(), Matchers.equalTo(2));
assertThat("some usages are populated", info.getNodeLeastAvailableDiskUsages().size(), Matchers.equalTo(2));
assertThat("some shard sizes are populated", info.shardSizes.size(), greaterThan(0));
MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, internalTestCluster.getMasterName());
final AtomicBoolean timeout = new AtomicBoolean(false);
final Set<String> blockedActions = ImmutableSet.of(NodesStatsAction.NAME, NodesStatsAction.NAME + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[s]");
final Set<String> blockedActions = ImmutableSet.of(NodesStatsAction.NAME, NodesStatsAction.NAME + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[n]");
// drop all outgoing stats requests to force a timeout.
for (DiscoveryNode node : internalTestCluster.clusterService().state().getNodes()) {
mockTransportService.addDelegate(node, new MockTransportService.DelegateTransport(mockTransportService.original()) {
@ -231,7 +265,8 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
// node info will time out both on the request level on the count down latch. this means
// it is likely to update the node disk usage based on the one response that came be from local
// node.
assertThat(info.getNodeDiskUsages().size(), greaterThanOrEqualTo(1));
assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThanOrEqualTo(1));
assertThat(info.getNodeMostAvailableDiskUsages().size(), greaterThanOrEqualTo(1));
// indices is guaranteed to time out on the latch, not updating anything.
assertThat(info.shardSizes.size(), greaterThan(1));
@ -252,7 +287,8 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
infoService.updateOnce();
info = listener.get();
assertNotNull("info should not be null", info);
assertThat(info.getNodeDiskUsages().size(), equalTo(0));
assertThat(info.getNodeLeastAvailableDiskUsages().size(), equalTo(0));
assertThat(info.getNodeMostAvailableDiskUsages().size(), equalTo(0));
assertThat(info.shardSizes.size(), equalTo(0));
// check we recover
@ -261,7 +297,8 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
infoService.updateOnce();
info = listener.get();
assertNotNull("info should not be null", info);
assertThat(info.getNodeDiskUsages().size(), equalTo(2));
assertThat(info.getNodeLeastAvailableDiskUsages().size(), equalTo(2));
assertThat(info.getNodeMostAvailableDiskUsages().size(), equalTo(2));
assertThat(info.shardSizes.size(), greaterThan(0));
}

6
core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java
View File

@ -69,6 +69,11 @@ import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.SuppressLocalMode
public class ClusterServiceIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(TestPlugin.class);
}
@Test
public void testTimeoutUpdateTask() throws Exception {
Settings settings = settingsBuilder()
@ -637,7 +642,6 @@ public class ClusterServiceIT extends ESIntegTestCase {
.put("discovery.zen.minimum_master_nodes", 1)
.put("discovery.zen.ping_timeout", "400ms")
.put("discovery.initial_state_timeout", "500ms")
.put("plugin.types", TestPlugin.class.getName())
.build();
String node_0 = internalCluster().startNode(settings);

108
core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
View File

@ -19,16 +19,33 @@
package org.elasticsearch.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingHelper;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.common.transport.DummyTransportAddress;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.test.ESTestCase;
import org.junit.Test;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
public class DiskUsageTests extends ESTestCase {
@Test
public void diskUsageCalcTest() {
DiskUsage du = new DiskUsage("node1", "n1", 100, 40);
DiskUsage du = new DiskUsage("node1", "n1", "random", 100, 40);
assertThat(du.getFreeDiskAsPercentage(), equalTo(40.0));
assertThat(du.getUsedDiskAsPercentage(), equalTo(100.0 - 40.0));
assertThat(du.getFreeBytes(), equalTo(40L));
@ -37,19 +54,19 @@ public class DiskUsageTests extends ESTestCase {
// Test that DiskUsage handles invalid numbers, as reported by some
// filesystems (ZFS & NTFS)
DiskUsage du2 = new DiskUsage("node1", "n1", 100, 101);
DiskUsage du2 = new DiskUsage("node1", "n1","random", 100, 101);
assertThat(du2.getFreeDiskAsPercentage(), equalTo(101.0));
assertThat(du2.getFreeBytes(), equalTo(101L));
assertThat(du2.getUsedBytes(), equalTo(-1L));
assertThat(du2.getTotalBytes(), equalTo(100L));
DiskUsage du3 = new DiskUsage("node1", "n1", -1, -1);
DiskUsage du3 = new DiskUsage("node1", "n1", "random",-1, -1);
assertThat(du3.getFreeDiskAsPercentage(), equalTo(100.0));
assertThat(du3.getFreeBytes(), equalTo(-1L));
assertThat(du3.getUsedBytes(), equalTo(0L));
assertThat(du3.getTotalBytes(), equalTo(-1L));
DiskUsage du4 = new DiskUsage("node1", "n1", 0, 0);
DiskUsage du4 = new DiskUsage("node1", "n1","random", 0, 0);
assertThat(du4.getFreeDiskAsPercentage(), equalTo(100.0));
assertThat(du4.getFreeBytes(), equalTo(0L));
assertThat(du4.getUsedBytes(), equalTo(0L));
@ -62,7 +79,7 @@ public class DiskUsageTests extends ESTestCase {
for (int i = 1; i < iters; i++) {
long total = between(Integer.MIN_VALUE, Integer.MAX_VALUE);
long free = between(Integer.MIN_VALUE, Integer.MAX_VALUE);
DiskUsage du = new DiskUsage("random", "random", total, free);
DiskUsage du = new DiskUsage("random", "random", "random", total, free);
if (total == 0) {
assertThat(du.getFreeBytes(), equalTo(free));
assertThat(du.getTotalBytes(), equalTo(0L));
@ -78,4 +95,85 @@ public class DiskUsageTests extends ESTestCase {
}
}
}
public void testFillShardLevelInfo() {
ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_0, "node1");
ShardRoutingHelper.moveToStarted(test_0);
Path test0Path = createTempDir().resolve("indices").resolve("test").resolve("0");
CommonStats commonStats0 = new CommonStats();
commonStats0.store = new StoreStats(100, 1);
ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_1, "node2");
ShardRoutingHelper.moveToStarted(test_1);
Path test1Path = createTempDir().resolve("indices").resolve("test").resolve("1");
CommonStats commonStats1 = new CommonStats();
commonStats1.store = new StoreStats(1000, 1);
ShardStats[] stats = new ShardStats[] {
new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0 , null),
new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1 , null)
};
HashMap<String, Long> shardSizes = new HashMap<>();
HashMap<ShardRouting, String> routingToPath = new HashMap<>();
InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath);
assertEquals(2, shardSizes.size());
assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_0)));
assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_1)));
assertEquals(100l, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_0)).longValue());
assertEquals(1000l, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_1)).longValue());
assertEquals(2, routingToPath.size());
assertTrue(routingToPath.containsKey(test_0));
assertTrue(routingToPath.containsKey(test_1));
assertEquals(test0Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_0));
assertEquals(test1Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_1));
}
public void testFillDiskUsage() {
Map<String, DiskUsage> newLeastAvaiableUsages = new HashMap<>();
Map<String, DiskUsage> newMostAvaiableUsages = new HashMap<>();
FsInfo.Path[] node1FSInfo = new FsInfo.Path[] {
new FsInfo.Path("/middle", "/dev/sda", 100, 90, 80),
new FsInfo.Path("/least", "/dev/sdb", 200, 190, 70),
new FsInfo.Path("/most", "/dev/sdc", 300, 290, 280),
};
FsInfo.Path[] node2FSInfo = new FsInfo.Path[] {
new FsInfo.Path("/least_most", "/dev/sda", 100, 90, 80),
};
FsInfo.Path[] node3FSInfo = new FsInfo.Path[] {
new FsInfo.Path("/least", "/dev/sda", 100, 90, 70),
new FsInfo.Path("/most", "/dev/sda", 100, 90, 80),
};
NodeStats[] nodeStats = new NodeStats[] {
new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null),
new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null),
new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null)
};
InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages);
DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1");
DiskUsage mostNode_1 = newMostAvaiableUsages.get("node_1");
assertDiskUsage(mostNode_1, node1FSInfo[2]);
assertDiskUsage(leastNode_1, node1FSInfo[1]);
DiskUsage leastNode_2 = newLeastAvaiableUsages.get("node_2");
DiskUsage mostNode_2 = newMostAvaiableUsages.get("node_2");
assertDiskUsage(leastNode_2, node2FSInfo[0]);
assertDiskUsage(mostNode_2, node2FSInfo[0]);
DiskUsage leastNode_3 = newLeastAvaiableUsages.get("node_3");
DiskUsage mostNode_3 = newMostAvaiableUsages.get("node_3");
assertDiskUsage(leastNode_3, node3FSInfo[0]);
assertDiskUsage(mostNode_3, node3FSInfo[1]);
}
private void assertDiskUsage(DiskUsage usage, FsInfo.Path path) {
assertEquals(usage.toString(), usage.getPath(), path.getPath());
assertEquals(usage.toString(), usage.getTotalBytes(), path.getTotal().bytes());
assertEquals(usage.toString(), usage.getFreeBytes(), path.getAvailable().bytes());
}
}

12
core/src/test/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDeciderTests;
import org.elasticsearch.cluster.routing.allocation.decider.MockDiskUsagesIT;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -63,9 +64,9 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
ClusterService clusterService, ThreadPool threadPool) {
super(settings, nodeSettingsService, transportNodesStatsAction, transportIndicesStatsAction, clusterService, threadPool);
this.clusterName = ClusterName.clusterNameFromSettings(settings);
stats[0] = MockDiskUsagesIT.makeStats("node_t1", new DiskUsage("node_t1", "n1", 100, 100));
stats[1] = MockDiskUsagesIT.makeStats("node_t2", new DiskUsage("node_t2", "n2", 100, 100));
stats[2] = MockDiskUsagesIT.makeStats("node_t3", new DiskUsage("node_t3", "n3", 100, 100));
stats[0] = MockDiskUsagesIT.makeStats("node_t1", new DiskUsage("node_t1", "n1", "/dev/null", 100, 100));
stats[1] = MockDiskUsagesIT.makeStats("node_t2", new DiskUsage("node_t2", "n2", "/dev/null", 100, 100));
stats[2] = MockDiskUsagesIT.makeStats("node_t3", new DiskUsage("node_t3", "n3", "/dev/null", 100, 100));
}
public void setN1Usage(String nodeName, DiskUsage newUsage) {
@ -92,4 +93,9 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
// Not used, so noop
return new CountDownLatch(0);
}
public ClusterInfo getClusterInfo() {
ClusterInfo clusterInfo = super.getClusterInfo();
return new ClusterInfo(clusterInfo.getNodeLeastAvailableDiskUsages(), clusterInfo.getNodeMostAvailableDiskUsages(), clusterInfo.shardSizes, DiskThresholdDeciderTests.DEV_NULL_MAP);
}
}

21
core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTest.java
View File

@ -243,4 +243,25 @@ public class RoutingTableTest extends ESAllocationTestCase {
fail("Calling with non-existing index should be ignored at the moment");
}
}
public void testAllShardsForMultipleIndices() {
assertThat(this.emptyRoutingTable.allShards(new String[0]).size(), is(0));
assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
initPrimaries();
assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
startInitializingShards(TEST_INDEX_1);
assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1}).size(), is(this.shardsPerIndex));
startInitializingShards(TEST_INDEX_2);
assertThat(this.testRoutingTable.allShards(new String[]{TEST_INDEX_1, TEST_INDEX_2}).size(), is(this.totalNumberOfShards));
try {
this.testRoutingTable.allShards(new String[]{TEST_INDEX_1, "not_exists"});
} catch (IndexNotFoundException e) {
fail("Calling with non-existing index should be ignored at the moment");
}
}
}

4
core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
View File

@ -59,7 +59,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
AllocationService strategy = createAllocationService(Settings.EMPTY, new ClusterInfoService() {
@Override
public ClusterInfo getClusterInfo() {
return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
return new ClusterInfo() {
@Override
public Long getShardSize(ShardRouting shardRouting) {
if (shardRouting.index().equals("test") && shardRouting.shardId().getId() == 0) {
@ -118,7 +118,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
final AllocationService allocation = createAllocationService(Settings.EMPTY, new ClusterInfoService() {
@Override
public ClusterInfo getClusterInfo() {
return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
return new ClusterInfo() {
@Override
public Long getShardSize(ShardRouting shardRouting) {
if (shardRouting.index().equals("test") && shardRouting.shardId().getId() == 0) {

2
core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
View File

@ -66,7 +66,7 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
new ClusterInfoService() {
@Override
public ClusterInfo getClusterInfo() {
return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
return new ClusterInfo() {
@Override
public Long getShardSize(ShardRouting shardRouting) {
if (shardRouting.index().equals("test")) {

85
core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
View File

@ -65,15 +65,15 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, 0.8).build();
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node1", new DiskUsage("node1", "node1", 100, 10)); // 90% used
usages.put("node2", new DiskUsage("node2", "node2", 100, 35)); // 65% used
usages.put("node3", new DiskUsage("node3", "node3", 100, 60)); // 40% used
usages.put("node4", new DiskUsage("node4", "node4", 100, 80)); // 20% used
usages.put("node1", new DiskUsage("node1", "node1", "/dev/null", 100, 10)); // 90% used
usages.put("node2", new DiskUsage("node2", "node2", "/dev/null", 100, 35)); // 65% used
usages.put("node3", new DiskUsage("node3", "node3", "/dev/null", 100, 60)); // 40% used
usages.put("node4", new DiskUsage("node4", "node4", "/dev/null", 100, 80)); // 20% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
shardSizes.put("[test][0][r]", 10L);
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
new HashSet<>(Arrays.asList(
@ -92,7 +92,6 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
// noop
}
};
AllocationService strategy = new AllocationService(settingsBuilder()
.put("cluster.routing.allocation.concurrent_recoveries", 10)
.put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always")
@ -259,16 +258,16 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "9b").build();
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node1", new DiskUsage("node1", "n1", 100, 10)); // 90% used
usages.put("node2", new DiskUsage("node2", "n2", 100, 10)); // 90% used
usages.put("node3", new DiskUsage("node3", "n3", 100, 60)); // 40% used
usages.put("node4", new DiskUsage("node4", "n4", 100, 80)); // 20% used
usages.put("node5", new DiskUsage("node5", "n5", 100, 85)); // 15% used
usages.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 10)); // 90% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 10)); // 90% used
usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 60)); // 40% used
usages.put("node4", new DiskUsage("node4", "n4", "/dev/null", 100, 80)); // 20% used
usages.put("node5", new DiskUsage("node5", "n5", "/dev/null", 100, 85)); // 15% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
shardSizes.put("[test][0][r]", 10L);
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
new HashSet<>(Arrays.asList(
@ -329,8 +328,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
logger.info("--> nodeWithoutPrimary: {}", nodeWithoutPrimary);
// Make node without the primary now habitable to replicas
usages.put(nodeWithoutPrimary, new DiskUsage(nodeWithoutPrimary, "", 100, 35)); // 65% used
final ClusterInfo clusterInfo2 = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
usages.put(nodeWithoutPrimary, new DiskUsage(nodeWithoutPrimary, "", "/dev/null", 100, 35)); // 65% used
final ClusterInfo clusterInfo2 = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
cis = new ClusterInfoService() {
@Override
public ClusterInfo getClusterInfo() {
@ -524,12 +523,12 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "71%").build();
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node1", new DiskUsage("node1", "n1", 100, 31)); // 69% used
usages.put("node2", new DiskUsage("node2", "n2", 100, 1)); // 99% used
usages.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 31)); // 69% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1)); // 99% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
new HashSet<>(Arrays.asList(
@ -590,13 +589,13 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, 0.85).build();
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node2", new DiskUsage("node2", "node2", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "node3", 100, 0)); // 100% used
usages.put("node2", new DiskUsage("node2", "node2", "/dev/null", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "node3", "/dev/null", 100, 0)); // 100% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
shardSizes.put("[test][0][r]", 10L); // 10 bytes
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
new HashSet<>(Arrays.asList(
@ -661,8 +660,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY);
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node2", new DiskUsage("node2", "n2", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "n3", 100, 0)); // 100% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0)); // 100% used
DiskUsage node1Usage = decider.averageUsage(rn, usages);
assertThat(node1Usage.getTotalBytes(), equalTo(100L));
@ -675,10 +674,10 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY);
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node2", new DiskUsage("node2", "n2", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "n3", 100, 0)); // 100% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 50)); // 50% used
usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0)); // 100% used
Double after = decider.freeDiskPercentageAfterShardAssigned(new DiskUsage("node2", "n2", 100, 30), 11L);
Double after = decider.freeDiskPercentageAfterShardAssigned(new DiskUsage("node2", "n2", "/dev/null", 100, 30), 11L);
assertThat(after, equalTo(19.0));
}
@ -691,16 +690,16 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, 0.8).build();
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node1", new DiskUsage("node1", "n1", 100, 40)); // 60% used
usages.put("node2", new DiskUsage("node2", "n2", 100, 40)); // 60% used
usages.put("node2", new DiskUsage("node3", "n3", 100, 40)); // 60% used
usages.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 40)); // 60% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 40)); // 60% used
usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 40)); // 60% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 14L); // 14 bytes
shardSizes.put("[test][0][r]", 14L);
shardSizes.put("[test2][0][p]", 1L); // 1 bytes
shardSizes.put("[test2][0][r]", 1L);
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
new HashSet<>(Arrays.asList(
@ -797,13 +796,13 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
// We have an index with 2 primary shards each taking 40 bytes. Each node has 100 bytes available
Map<String, DiskUsage> usages = new HashMap<>();
usages.put("node1", new DiskUsage("node1", "n1", 100, 20)); // 80% used
usages.put("node2", new DiskUsage("node2", "n2", 100, 100)); // 0% used
usages.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 20)); // 80% used
usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 100)); // 0% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 40L);
shardSizes.put("[test][1][p]", 40L);
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes));
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(usages), Collections.unmodifiableMap(usages), Collections.unmodifiableMap(shardSizes), DEV_NULL_MAP);
DiskThresholdDecider diskThresholdDecider = new DiskThresholdDecider(diskSettings);
MetaData metaData = MetaData.builder()
@ -916,4 +915,26 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
rn.shardsWithState(RELOCATING),
rn.shardsWithState(STARTED));
}
public static final Map<ShardRouting, String> DEV_NULL_MAP = Collections.unmodifiableMap(new StaticValueMap("/dev/null"));
// a test only map that always returns the same value no matter what key is passed
private static final class StaticValueMap extends AbstractMap<ShardRouting, String> {
private final String value;
private StaticValueMap(String value) {
this.value = value;
}
@Override
public String get(Object key) {
return value;
}
@Override
public Set<Entry<ShardRouting, String>> entrySet() {
throw new UnsupportedOperationException("this is a test-only map that only supports #get(Object key)");
}
}
}

154
core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
View File

@ -20,15 +20,15 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterInfoService;
import org.elasticsearch.cluster.EmptyClusterInfoService;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingHelper;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.DummyTransportAddress;
import org.elasticsearch.common.transport.LocalTransportAddress;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.node.settings.NodeSettingsService;
@ -76,11 +76,11 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
applySettings.onRefreshSettings(newSettings);
assertThat("high threshold bytes should be unset",
decider.getFreeBytesThresholdHigh(), equalTo(ByteSizeValue.parseBytesSizeValue("0b", "test")));
decider.getFreeBytesThresholdHigh(), equalTo(ByteSizeValue.parseBytesSizeValue("0b", "test")));
assertThat("high threshold percentage should be changed",
decider.getFreeDiskThresholdHigh(), equalTo(30.0d));
assertThat("low threshold bytes should be set to 500mb",
decider.getFreeBytesThresholdLow(), equalTo(ByteSizeValue.parseBytesSizeValue("500mb", "test")));
decider.getFreeBytesThresholdLow(), equalTo(ByteSizeValue.parseBytesSizeValue("500mb", "test")));
assertThat("low threshold bytes should be unset",
decider.getFreeDiskThresholdLow(), equalTo(0.0d));
assertThat("reroute interval should be changed to 30 seconds",
@ -89,13 +89,133 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
assertFalse("relocations should now be disabled", decider.isIncludeRelocations());
}
public void testCanAllocateUsesMaxAvailableSpace() {
NodeSettingsService nss = new NodeSettingsService(Settings.EMPTY);
ClusterInfoService cis = EmptyClusterInfoService.INSTANCE;
DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss, cis, null);
ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Version.CURRENT);
DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT);
MetaData metaData = MetaData.builder()
.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder()
.addAsNew(metaData.index("test"))
.build();
ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
.put(node_0)
.put(node_1)
).build();
// actual test -- after all that bloat :)
Map<String, DiskUsage> leastAvailableUsages = new HashMap<>();
leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0)); // all full
leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, 0)); // all full
Map<String, DiskUsage> mostAvailableUsage = new HashMap<>();
mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, randomIntBetween(20, 100))); // 20 - 99 percent since after allocation there must be at least 10% left and shard is 10byte
mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, randomIntBetween(0, 10))); // this is weird and smells like a bug! it should be up to 20%?
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(leastAvailableUsages), Collections.unmodifiableMap(mostAvailableUsage), Collections.unmodifiableMap(shardSizes), Collections.EMPTY_MAP);
RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo);
assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation));
assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation));
}
public void testCanRemainUsesLeastAvailableSpace() {
NodeSettingsService nss = new NodeSettingsService(Settings.EMPTY);
ClusterInfoService cis = EmptyClusterInfoService.INSTANCE;
DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss, cis, null);
Map<ShardRouting, String> shardRoutingMap = new HashMap<>();
DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Version.CURRENT);
DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT);
ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_0, node_0.getId());
ShardRoutingHelper.moveToStarted(test_0);
shardRoutingMap.put(test_0, "/node0/least");
ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_1, node_1.getId());
ShardRoutingHelper.moveToStarted(test_1);
shardRoutingMap.put(test_1, "/node1/least");
MetaData metaData = MetaData.builder()
.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder()
.addAsNew(metaData.index("test"))
.build();
ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
logger.info("--> adding two nodes");
clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
.put(node_0)
.put(node_1)
).build();
// actual test -- after all that bloat :)
Map<String, DiskUsage> leastAvailableUsages = new HashMap<>();
leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "/node0/least", 100, 10)); // 90% used
leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "/node1/least", 100, 9)); // 91% used
Map<String, DiskUsage> mostAvailableUsage = new HashMap<>();
mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "/node0/most", 100, 90)); // 10% used
mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "/node1/most", 100, 90)); // 10% used
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][p]", 10L); // 10 bytes
shardSizes.put("[test][1][p]", 10L);
shardSizes.put("[test][2][p]", 10L);
final ClusterInfo clusterInfo = new ClusterInfo(Collections.unmodifiableMap(leastAvailableUsages), Collections.unmodifiableMap(mostAvailableUsage), Collections.unmodifiableMap(shardSizes), shardRoutingMap);
RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo);
assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation));
assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation));
try {
decider.canRemain(test_0, new RoutingNode("node_1", node_1), allocation);
fail("not allocated on this node");
} catch (IllegalArgumentException ex) {
// not allocated on that node
}
try {
decider.canRemain(test_1, new RoutingNode("node_0", node_0), allocation);
fail("not allocated on this node");
} catch (IllegalArgumentException ex) {
// not allocated on that node
}
ShardRouting test_2 = ShardRouting.newUnassigned("test", 2, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_2, node_1.getId());
ShardRoutingHelper.moveToStarted(test_2);
shardRoutingMap.put(test_2, "/node1/most");
assertEquals("can stay since allocated on a different path with enough space", Decision.YES, decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation));
ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_3, node_1.getId());
ShardRoutingHelper.moveToStarted(test_3);
assertEquals("can stay since we don't have information about this shard", Decision.YES, decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation));
}
public void testShardSizeAndRelocatingSize() {
Map<String, Long> shardSizes = new HashMap<>();
shardSizes.put("[test][0][r]", 10L);
shardSizes.put("[test][1][r]", 100L);
shardSizes.put("[test][2][r]", 1000L);
shardSizes.put("[other][0][p]", 10000L);
ClusterInfo info = new ClusterInfo(Collections.EMPTY_MAP, shardSizes);
ClusterInfo info = new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP, shardSizes, DiskThresholdDeciderTests.DEV_NULL_MAP);
ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_0, "node1");
ShardRoutingHelper.moveToStarted(test_0);
@ -115,8 +235,10 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
assertEquals(10l, DiskThresholdDecider.getShardSize(test_0, info));
RoutingNode node = new RoutingNode("node1", new DiscoveryNode("node1", LocalTransportAddress.PROTO, Version.CURRENT), Arrays.asList(test_0, test_1.buildTargetRelocatingShard(), test_2));
assertEquals(100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false));
assertEquals(90l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true));
assertEquals(100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
assertEquals(90l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
assertEquals(0l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
assertEquals(0l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_3, "node1");
@ -132,11 +254,11 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
node = new RoutingNode("node1", new DiscoveryNode("node1", LocalTransportAddress.PROTO, Version.CURRENT), Arrays.asList(test_0, test_1.buildTargetRelocatingShard(), test_2, other_0.buildTargetRelocatingShard()));
if (other_0.primary()) {
assertEquals(10100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false));
assertEquals(10090l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true));
assertEquals(10100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
assertEquals(10090l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
} else {
assertEquals(100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false));
assertEquals(90l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true));
assertEquals(100l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
assertEquals(90l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
}
}

34
core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
View File

@ -33,9 +33,11 @@ import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.DummyTransportAddress;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Collection;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -52,13 +54,17 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
// Use the mock internal cluster info service, which has fake-able disk usages
.extendArray("plugin.types", MockInternalClusterInfoService.TestPlugin.class.getName())
// Update more frequently
.put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, "1s")
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
// Use the mock internal cluster info service, which has fake-able disk usages
return pluginList(MockInternalClusterInfoService.TestPlugin.class);
}
@Test
//@TestLogging("org.elasticsearch.cluster:TRACE,org.elasticsearch.cluster.routing.allocation.decider:TRACE")
public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
@ -76,9 +82,9 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
// Start with all nodes at 50% usage
final MockInternalClusterInfoService cis = (MockInternalClusterInfoService)
internalCluster().getInstance(ClusterInfoService.class, internalCluster().getMasterName());
cis.setN1Usage(nodes.get(0), new DiskUsage(nodes.get(0), "n1", 100, 50));
cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", 100, 50));
cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", 100, 50));
cis.setN1Usage(nodes.get(0), new DiskUsage(nodes.get(0), "n1", "/dev/null", 100, 50));
cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", "/dev/null", 100, 50));
cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", "/dev/null", 100, 50));
client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder()
.put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, randomFrom("20b", "80%"))
@ -97,8 +103,8 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
@Override
public void run() {
ClusterInfo info = cis.getClusterInfo();
logger.info("--> got: {} nodes", info.getNodeDiskUsages().size());
assertThat(info.getNodeDiskUsages().size(), greaterThan(0));
logger.info("--> got: {} nodes", info.getNodeLeastAvailableDiskUsages().size());
assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThan(0));
}
});
@ -113,9 +119,9 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
}
// Update the disk usages so one node has now passed the high watermark
cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", 100, 50));
cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", 100, 50));
cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", 100, 0)); // nothing free on node3
cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 50));
cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 0)); // nothing free on node3
// Retrieve the count of shards on each node
final Map<String, Integer> nodesToShardCount = newHashMap();
@ -138,9 +144,9 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
});
// Update the disk usages so one node is now back under the high watermark
cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", 100, 50));
cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", 100, 50));
cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", 100, 50)); // node3 has free space now
cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 50));
cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 50)); // node3 has free space now
// Retrieve the count of shards on each node
nodesToShardCount.clear();
@ -166,7 +172,7 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
/** Create a fake NodeStats for the given node and usage */
public static NodeStats makeStats(String nodeName, DiskUsage usage) {
FsInfo.Path[] paths = new FsInfo.Path[1];
FsInfo.Path path = new FsInfo.Path("/path.data", null,
FsInfo.Path path = new FsInfo.Path("/dev/null", null,
usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes());
paths[0] = path;
FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), paths);

7
core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
View File

@ -42,11 +42,8 @@ import static org.hamcrest.Matchers.nullValue;
public class SettingsFilteringIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", SettingsFilteringPlugin.class.getName())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(SettingsFilteringPlugin.class);
}
public static class SettingsFilteringPlugin extends Plugin {

9
core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
View File

@ -48,6 +48,7 @@ import org.elasticsearch.discovery.zen.ping.ZenPing;
import org.elasticsearch.discovery.zen.ping.ZenPingService;
import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
@ -82,7 +83,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return discoveryConfig.node(nodeOrdinal);
return discoveryConfig.nodeSettings(nodeOrdinal);
}
@Before
@ -134,9 +135,13 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
.put("transport.bind_host", "127.0.0.1")
.put("transport.publish_host", "127.0.0.1")
.put("gateway.local.list_timeout", "10s") // still long to induce failures but to long so test won't time out
.put("plugin.types", MockTransportService.TestPlugin.class.getName())
.build();
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
private void configureUnicastCluster(int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws ExecutionException, InterruptedException {
if (minimumMasterNode < 0) {
minimumMasterNode = numberOfNodes / 2 + 1;

2
core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java
View File

@ -43,7 +43,7 @@ public class ZenUnicastDiscoveryIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return discoveryConfig.node(nodeOrdinal);
return discoveryConfig.nodeSettings(nodeOrdinal);
}
@Before

10
core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
View File

@ -42,7 +42,11 @@ import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
import org.elasticsearch.transport.BytesTransportRequest;
import org.elasticsearch.transport.EmptyTransportResponseHandler;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportService;
import org.hamcrest.Matchers;
import org.junit.Test;
@ -112,7 +116,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
createIndex("test");
ensureSearchable("test");
RecoveryResponse r = client().admin().indices().prepareRecoveries("test").get();
int numRecoveriesBeforeNewMaster = r.shardResponses().get("test").size();
int numRecoveriesBeforeNewMaster = r.shardRecoveryStates().get("test").size();
final String oldMaster = internalCluster().getMasterName();
internalCluster().stopCurrentMasterNode();
@ -127,7 +131,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
ensureSearchable("test");
r = client().admin().indices().prepareRecoveries("test").get();
int numRecoveriesAfterNewMaster = r.shardResponses().get("test").size();
int numRecoveriesAfterNewMaster = r.shardRecoveryStates().get("test").size();
assertThat(numRecoveriesAfterNewMaster, equalTo(numRecoveriesBeforeNewMaster));
}

7
core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
View File

@ -35,7 +35,10 @@ import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
/**
*/
@ -146,7 +149,7 @@ public class ShardInfoIT extends ESIntegTestCase {
RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("idx")
.setActiveOnly(true)
.get();
assertThat(recoveryResponse.shardResponses().get("idx").size(), equalTo(0));
assertThat(recoveryResponse.shardRecoveryStates().get("idx").size(), equalTo(0));
}
});
}

4
core/src/test/java/org/elasticsearch/gateway/RecoveryBackwardsCompatibilityIT.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.gateway;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -99,8 +98,7 @@ public class RecoveryBackwardsCompatibilityIT extends ESBackcompatTestCase {
HashMap<String, String> map = new HashMap<>();
map.put("details", "true");
final ToXContent.Params params = new ToXContent.MapParams(map);
for (ShardRecoveryResponse response : recoveryResponse.shardResponses().get("test")) {
RecoveryState recoveryState = response.recoveryState();
for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
final String recoverStateAsJSON = XContentHelper.toString(recoveryState, params);
if (!recoveryState.getPrimary()) {
RecoveryState.Index index = recoveryState.getIndex();

8
core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.gateway;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.client.Client;
@ -400,8 +399,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
assertSyncIdsNotNull();
}
RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
for (ShardRecoveryResponse response : recoveryResponse.shardResponses().get("test")) {
RecoveryState recoveryState = response.recoveryState();
for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
long recovered = 0;
for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
if (file.name().startsWith("segments")) {
@ -410,7 +408,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
}
if (!recoveryState.getPrimary() && (useSyncIds == false)) {
logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}",
response.getShardId(), recoveryState.getSourceNode().name(), recoveryState.getTargetNode().name(),
recoveryState.getShardId().getId(), recoveryState.getSourceNode().name(), recoveryState.getTargetNode().name(),
recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
assertThat("no bytes should be recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0l));
@ -422,7 +420,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
} else {
if (useSyncIds && !recoveryState.getPrimary()) {
logger.info("--> replica shard {} recovered from {} to {} using sync id, recovered {}, reuse {}",
response.getShardId(), recoveryState.getSourceNode().name(), recoveryState.getTargetNode().name(),
recoveryState.getShardId().getId(), recoveryState.getSourceNode().name(), recoveryState.getTargetNode().name(),
recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
}
assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0l));

8
core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShadowIndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoveryTarget;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.snapshots.SnapshotState;
@ -52,6 +53,7 @@ import org.junit.Test;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
@ -83,6 +85,11 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
public void testCannotCreateWithBadPath() throws Exception {
Settings nodeSettings = nodeSettings("/badpath");
internalCluster().startNodesAsync(1, nodeSettings).get();
@ -419,7 +426,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
Path dataPath = createTempDir();
Settings nodeSettings = Settings.builder()
.put("node.add_id_to_custom_path", false)
.put("plugin.types", MockTransportService.TestPlugin.class.getName())
.put("path.shared_data", dataPath)
.build();

8
core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java
View File

@ -33,12 +33,14 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.fd.FaultDetection;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportService;
import org.junit.Test;
import java.util.Collection;
import java.util.List;
import static org.elasticsearch.cluster.routing.ShardRoutingState.*;
@ -57,9 +59,13 @@ public class TransportIndexFailuresIT extends ESIntegTestCase {
.put(FaultDetection.SETTING_PING_RETRIES, "1") // <-- for hitting simulated network failures quickly
.put(DiscoverySettings.PUBLISH_TIMEOUT, "1s") // <-- for hitting simulated network failures quickly
.put("discovery.zen.minimum_master_nodes", 1)
.put("plugin.types", MockTransportService.TestPlugin.class.getName())
.build();
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
@Override
protected int numberOfShards() {
return 1;

11
core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
View File

@ -19,12 +19,9 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.json.JsonXContentParser;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.test.ESSingleNodeTestCase;
// TODO: make this a real unit test
@ -37,11 +34,12 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
DocumentMapper mapper = mapperParser.parse(mapping);
BytesReference bytes = XContentFactory.jsonBuilder()
.startObject()
.startObject().startObject("foo")
.field("field", "1234")
.endObject().bytes();
.endObject().endObject().bytes();
ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
assertNull(doc.rootDoc().getField("field"));
assertNotNull(doc.rootDoc().getField(UidFieldMapper.NAME));
}
public void testFieldDisabled() throws Exception {
@ -60,5 +58,6 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
assertNull(doc.rootDoc().getField("foo"));
assertNotNull(doc.rootDoc().getField("bar"));
assertNotNull(doc.rootDoc().getField(UidFieldMapper.NAME));
}
}

11
core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java
View File

@ -25,19 +25,20 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collection;
import static org.hamcrest.Matchers.equalTo;
public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", ExternalMapperPlugin.class.getName())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(ExternalMapperPlugin.class);
}
@Test

8
core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
View File

@ -27,10 +27,13 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import org.junit.Test;
import java.util.Collection;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@ -39,9 +42,8 @@ import static org.hamcrest.Matchers.instanceOf;
public class CustomQueryParserIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", DummyQueryParserPlugin.class.getName()).build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(DummyQueryParserPlugin.class);
}
@Before

8
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
@ -67,7 +68,10 @@ import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
import static org.elasticsearch.cluster.metadata.IndexMetaData.EMPTY_PARAMS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@ -555,7 +559,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService("test");
IndexShard shard = test.shard(0);
ShardStats stats = new ShardStats(shard, new CommonStatsFlags());
ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(shard, new CommonStatsFlags()), shard.commitStats());
assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());

236
core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTest.java Normal file
View File

@ -0,0 +1,236 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.shard;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
import org.apache.lucene.mockfile.FilterFileSystem;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
import org.apache.lucene.mockfile.FilterPath;
import org.apache.lucene.util.Constants;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment.NodePath;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.FileStore;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.attribute.FileStoreAttributeView;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
/** Separate test class from ShardPathTests because we need static (BeforeClass) setup to install mock filesystems... */
@SuppressForbidden(reason = "ProviderMismatchException if I try to use PathUtils.getDefault instead")
public class NewPathForShardTest extends ESTestCase {
// Sneakiness to install mock file stores so we can pretend how much free space we have on each path.data:
private static MockFileStore aFileStore = new MockFileStore("mocka");
private static MockFileStore bFileStore = new MockFileStore("mockb");
private static FileSystem origFileSystem;
private static String aPathPart = File.separator + 'a' + File.separator;
private static String bPathPart = File.separator + 'b' + File.separator;
@BeforeClass
public static void installMockUsableSpaceFS() throws Exception {
// Necessary so when Environment.clinit runs, to gather all FileStores, it sees ours:
origFileSystem = FileSystems.getDefault();
Field field = PathUtils.class.getDeclaredField("DEFAULT");
field.setAccessible(true);
FileSystem mock = new MockUsableSpaceFileSystemProvider().getFileSystem(getBaseTempDirForTestClass().toUri());
field.set(null, mock);
assertEquals(mock, PathUtils.getDefaultFileSystem());
}
@AfterClass
public static void removeMockUsableSpaceFS() throws Exception {
Field field = PathUtils.class.getDeclaredField("DEFAULT");
field.setAccessible(true);
field.set(null, origFileSystem);
origFileSystem = null;
aFileStore = null;
bFileStore = null;
}
/** Mock file system that fakes usable space for each FileStore */
@SuppressForbidden(reason = "ProviderMismatchException if I try to use PathUtils.getDefault instead")
static class MockUsableSpaceFileSystemProvider extends FilterFileSystemProvider {
public MockUsableSpaceFileSystemProvider() {
super("mockusablespace://", FileSystems.getDefault());
final List<FileStore> fileStores = new ArrayList<>();
fileStores.add(aFileStore);
fileStores.add(bFileStore);
fileSystem = new FilterFileSystem(this, origFileSystem) {
@Override
public Iterable<FileStore> getFileStores() {
return fileStores;
}
};
}
@Override
public FileStore getFileStore(Path path) throws IOException {
if (path.toString().contains(aPathPart)) {
return aFileStore;
} else {
return bFileStore;
}
}
}
static class MockFileStore extends FileStore {
public long usableSpace;
private final String desc;
public MockFileStore(String desc) {
this.desc = desc;
}
@Override
public String type() {
return "mock";
}
@Override
public String name() {
return desc;
}
@Override
public String toString() {
return desc;
}
@Override
public boolean isReadOnly() {
return false;
}
@Override
public long getTotalSpace() throws IOException {
return usableSpace*3;
}
@Override
public long getUsableSpace() throws IOException {
return usableSpace;
}
@Override
public long getUnallocatedSpace() throws IOException {
return usableSpace*2;
}
@Override
public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {
return false;
}
@Override
public boolean supportsFileAttributeView(String name) {
return false;
}
@Override
public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {
return null;
}
@Override
public Object getAttribute(String attribute) throws IOException {
return null;
}
}
public void testSelectNewPathForShard() throws Exception {
assumeFalse("Consistenty fails on windows ('could not remove the following files')", Constants.WINDOWS);
Path path = PathUtils.get(createTempDir().toString());
// Use 2 data paths:
String[] paths = new String[] {path.resolve("a").toString(),
path.resolve("b").toString()};
Settings settings = Settings.builder()
.put("path.home", path)
.putArray("path.data", paths).build();
NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings));
// Make sure all our mocking above actually worked:
NodePath[] nodePaths = nodeEnv.nodePaths();
assertEquals(2, nodePaths.length);
assertEquals("mocka", nodePaths[0].fileStore.name());
assertEquals("mockb", nodePaths[1].fileStore.name());
// Path a has lots of free space, but b has little, so new shard should go to a:
aFileStore.usableSpace = 100000;
bFileStore.usableSpace = 1000;
ShardId shardId = new ShardId("index", 0);
ShardPath result = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path,Integer>emptyMap());
assertTrue(result.getDataPath().toString().contains(aPathPart));
// Test the reverse: b has lots of free space, but a has little, so new shard should go to b:
aFileStore.usableSpace = 1000;
bFileStore.usableSpace = 100000;
shardId = new ShardId("index", 0);
result = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path,Integer>emptyMap());
assertTrue(result.getDataPath().toString().contains(bPathPart));
// Now a and be have equal usable space; we allocate two shards to the node, and each should go to different paths:
aFileStore.usableSpace = 100000;
bFileStore.usableSpace = 100000;
Map<Path,Integer> dataPathToShardCount = new HashMap<>();
ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount);
dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1);
ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount);
// #11122: this was the original failure: on a node with 2 disks that have nearly equal
// free space, we would always allocate all N incoming shards to the one path that
// had the most free space, never using the other drive unless new shards arrive
// after the first shards started using storage:
assertNotEquals(result1.getDataPath(), result2.getDataPath());
}
}

8
core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
View File

@ -70,6 +70,7 @@ import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.recovery.RecoveryTarget;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
@ -91,6 +92,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -117,7 +119,6 @@ public class CorruptedFileIT extends ESIntegTestCase {
// we really need local GW here since this also checks for corruption etc.
// and we need to make sure primaries are not just trashed if we don't have replicas
.put(super.nodeSettings(nodeOrdinal))
.extendArray("plugin.types", MockTransportService.TestPlugin.class.getName())
// speed up recoveries
.put(RecoverySettings.INDICES_RECOVERY_CONCURRENT_STREAMS, 10)
.put(RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS, 10)
@ -125,6 +126,11 @@ public class CorruptedFileIT extends ESIntegTestCase {
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
/**
* Tests that we can actually recover from a corruption on the primary given that we have replica shards around.
*/

12
core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.translog.TranslogConfig;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.engine.MockEngineSupport;
import org.elasticsearch.test.junit.annotations.TestLogging;
@ -46,6 +47,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
@ -62,12 +64,10 @@ import static org.hamcrest.Matchers.notNullValue;
public class CorruptedTranslogIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
// we really need local GW here since this also checks for corruption etc.
// and we need to make sure primaries are not just trashed if we don't have replicas
.put(super.nodeSettings(nodeOrdinal))
.extendArray("plugin.types", MockTransportService.TestPlugin.class.getName()).build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
// we really need local GW here since this also checks for corruption etc.
// and we need to make sure primaries are not just trashed if we don't have replicas
return pluginList(MockTransportService.TestPlugin.class);
}
@Test

9
core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.transport.MockTransportService;
@ -36,6 +37,7 @@ import org.elasticsearch.transport.*;
import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@ -51,11 +53,8 @@ import static org.hamcrest.Matchers.greaterThan;
public class ExceptionRetryIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.extendArray("plugin.types", MockTransportService.TestPlugin.class.getName())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
@Override

9
core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
View File

@ -24,11 +24,13 @@ import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESBackcompatTestCase;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.lang.reflect.Field;
import java.util.Collection;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
@ -44,11 +46,8 @@ import static org.hamcrest.Matchers.notNullValue;
public class PreBuiltAnalyzerIntegrationIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", DummyAnalysisPlugin.class.getName())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(DummyAnalysisPlugin.class);
}
@Test

7
core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
View File

@ -45,6 +45,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@ -57,6 +58,11 @@ import static org.hamcrest.Matchers.equalTo;
*/
public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class);
}
@Test
public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
for (NodeStats node : client().admin().cluster().prepareNodesStats()
@ -107,7 +113,6 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
Settings.Builder settings = settingsBuilder()
.put(indexSettings())
.extendArray("plugin.types", RandomExceptionDirectoryReaderWrapper.TestPlugin.class.getName())
.put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate)
.put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate)
.put(MockEngineSupport.WRAP_READER_RATIO, 1.0d);

160
core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
@ -47,17 +46,24 @@ import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoveryState.Stage;
import org.elasticsearch.indices.recovery.RecoveryState.Type;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.store.MockFSDirectoryService;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.*;
import org.elasticsearch.transport.ConnectTransportException;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestOptions;
import org.elasticsearch.transport.TransportService;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@ -67,7 +73,13 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.arrayWithSize;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.not;
/**
*
@ -85,6 +97,10 @@ public class IndexRecoveryIT extends ESIntegTestCase {
private static final int SHARD_COUNT = 1;
private static final int REPLICA_COUNT = 0;
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
private void assertRecoveryStateWithoutStage(RecoveryState state, int shardId, Type type,
String sourceNode, String targetNode, boolean hasRestoreSource) {
@ -155,18 +171,17 @@ public class IndexRecoveryIT extends ESIntegTestCase {
logger.info("--> request recoveries");
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
assertThat(response.shardResponses().size(), equalTo(SHARD_COUNT));
assertThat(response.shardResponses().get(INDEX_NAME).size(), equalTo(1));
assertThat(response.shardRecoveryStates().size(), equalTo(SHARD_COUNT));
assertThat(response.shardRecoveryStates().get(INDEX_NAME).size(), equalTo(1));
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(1));
List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
assertThat(recoveryStates.size(), equalTo(1));
ShardRecoveryResponse shardResponse = shardResponses.get(0);
RecoveryState state = shardResponse.recoveryState();
RecoveryState recoveryState = recoveryStates.get(0);
assertRecoveryState(state, 0, Type.STORE, Stage.DONE, node, node, false);
assertRecoveryState(recoveryState, 0, Type.STORE, Stage.DONE, node, node, false);
validateIndexRecoveryState(state.getIndex());
validateIndexRecoveryState(recoveryState.getIndex());
}
@Test
@ -183,8 +198,8 @@ public class IndexRecoveryIT extends ESIntegTestCase {
logger.info("--> request recoveries");
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).setActiveOnly(true).execute().actionGet();
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(0)); // Should not expect any responses back
List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
assertThat(recoveryStates.size(), equalTo(0)); // Should not expect any responses back
}
@Test
@ -209,23 +224,23 @@ public class IndexRecoveryIT extends ESIntegTestCase {
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
// we should now have two total shards, one primary and one replica
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(2));
List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
assertThat(recoveryStates.size(), equalTo(2));
List<ShardRecoveryResponse> nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
List<RecoveryState> nodeAResponses = findRecoveriesForTargetNode(nodeA, recoveryStates);
assertThat(nodeAResponses.size(), equalTo(1));
List<ShardRecoveryResponse> nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
List<RecoveryState> nodeBResponses = findRecoveriesForTargetNode(nodeB, recoveryStates);
assertThat(nodeBResponses.size(), equalTo(1));
// validate node A recovery
ShardRecoveryResponse nodeAShardResponse = nodeAResponses.get(0);
assertRecoveryState(nodeAShardResponse.recoveryState(), 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
validateIndexRecoveryState(nodeAShardResponse.recoveryState().getIndex());
RecoveryState nodeARecoveryState = nodeAResponses.get(0);
assertRecoveryState(nodeARecoveryState, 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
validateIndexRecoveryState(nodeARecoveryState.getIndex());
// validate node B recovery
ShardRecoveryResponse nodeBShardResponse = nodeBResponses.get(0);
assertRecoveryState(nodeBShardResponse.recoveryState(), 0, Type.REPLICA, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBShardResponse.recoveryState().getIndex());
RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
assertRecoveryState(nodeBRecoveryState, 0, Type.REPLICA, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBRecoveryState.getIndex());
}
@Test
@ -266,17 +281,17 @@ public class IndexRecoveryIT extends ESIntegTestCase {
logger.info("--> request recoveries");
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
List<ShardRecoveryResponse> nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
assertThat(nodeAResponses.size(), equalTo(1));
List<ShardRecoveryResponse> nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
assertThat(nodeBResponses.size(), equalTo(1));
List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
assertThat(nodeARecoveryStates.size(), equalTo(1));
List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
assertThat(nodeBRecoveryStates.size(), equalTo(1));
assertRecoveryState(nodeAResponses.get(0).recoveryState(), 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
validateIndexRecoveryState(nodeAResponses.get(0).recoveryState().getIndex());
assertRecoveryState(nodeARecoveryStates.get(0), 0, Type.STORE, Stage.DONE, nodeA, nodeA, false);
validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
assertOnGoingRecoveryState(nodeBResponses.get(0).recoveryState(), 0, Type.RELOCATION, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBResponses.get(0).recoveryState().getIndex());
assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, Type.RELOCATION, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
logger.info("--> request node recovery stats");
NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
@ -325,11 +340,11 @@ public class IndexRecoveryIT extends ESIntegTestCase {
response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(1));
recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
assertThat(recoveryStates.size(), equalTo(1));
assertRecoveryState(shardResponses.get(0).recoveryState(), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(shardResponses.get(0).recoveryState().getIndex());
assertRecoveryState(recoveryStates.get(0), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(recoveryStates.get(0).getIndex());
statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
assertThat(statsResponse.getNodes(), arrayWithSize(2));
@ -377,45 +392,45 @@ public class IndexRecoveryIT extends ESIntegTestCase {
.execute().actionGet().getState();
response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
shardResponses = response.shardResponses().get(INDEX_NAME);
recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
assertThat(nodeAResponses.size(), equalTo(1));
nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
assertThat(nodeBResponses.size(), equalTo(1));
List<ShardRecoveryResponse> nodeCResponses = findRecoveriesForTargetNode(nodeC, shardResponses);
assertThat(nodeCResponses.size(), equalTo(1));
nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
assertThat(nodeARecoveryStates.size(), equalTo(1));
nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
assertThat(nodeBRecoveryStates.size(), equalTo(1));
List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
assertThat(nodeCRecoveryStates.size(), equalTo(1));
assertRecoveryState(nodeAResponses.get(0).recoveryState(), 0, Type.REPLICA, Stage.DONE, nodeB, nodeA, false);
validateIndexRecoveryState(nodeAResponses.get(0).recoveryState().getIndex());
assertRecoveryState(nodeARecoveryStates.get(0), 0, Type.REPLICA, Stage.DONE, nodeB, nodeA, false);
validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
assertRecoveryState(nodeBResponses.get(0).recoveryState(), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBResponses.get(0).recoveryState().getIndex());
assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
// relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
assertOnGoingRecoveryState(nodeCResponses.get(0).recoveryState(), 0, Type.REPLICA, nodeB, nodeC, false);
validateIndexRecoveryState(nodeCResponses.get(0).recoveryState().getIndex());
assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, nodeB, nodeC, false);
validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
logger.info("--> speeding up recoveries");
restoreRecoverySpeed();
ensureGreen();
response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
shardResponses = response.shardResponses().get(INDEX_NAME);
recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
assertThat(nodeAResponses.size(), equalTo(0));
nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
assertThat(nodeBResponses.size(), equalTo(1));
nodeCResponses = findRecoveriesForTargetNode(nodeC, shardResponses);
assertThat(nodeCResponses.size(), equalTo(1));
nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
assertThat(nodeARecoveryStates.size(), equalTo(0));
nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
assertThat(nodeBRecoveryStates.size(), equalTo(1));
nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
assertThat(nodeCRecoveryStates.size(), equalTo(1));
assertRecoveryState(nodeBResponses.get(0).recoveryState(), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBResponses.get(0).recoveryState().getIndex());
assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.RELOCATION, Stage.DONE, nodeA, nodeB, false);
validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
// relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
assertRecoveryState(nodeCResponses.get(0).recoveryState(), 0, Type.REPLICA, Stage.DONE, nodeB, nodeC, false);
validateIndexRecoveryState(nodeCResponses.get(0).recoveryState().getIndex());
assertRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, Stage.DONE, nodeB, nodeC, false);
validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}
@Test
@ -457,24 +472,24 @@ public class IndexRecoveryIT extends ESIntegTestCase {
logger.info("--> request recoveries");
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
for (Map.Entry<String, List<ShardRecoveryResponse>> shardRecoveryResponse : response.shardResponses().entrySet()) {
for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
assertThat(shardRecoveryResponse.getKey(), equalTo(INDEX_NAME));
List<ShardRecoveryResponse> shardRecoveryResponses = shardRecoveryResponse.getValue();
assertThat(shardRecoveryResponses.size(), equalTo(totalShards));
assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
assertThat(recoveryStates.size(), equalTo(totalShards));
for (ShardRecoveryResponse shardResponse : shardRecoveryResponses) {
assertRecoveryState(shardResponse.recoveryState(), 0, Type.SNAPSHOT, Stage.DONE, null, nodeA, true);
validateIndexRecoveryState(shardResponse.recoveryState().getIndex());
for (RecoveryState recoveryState : recoveryStates) {
assertRecoveryState(recoveryState, 0, Type.SNAPSHOT, Stage.DONE, null, nodeA, true);
validateIndexRecoveryState(recoveryState.getIndex());
}
}
}
private List<ShardRecoveryResponse> findRecoveriesForTargetNode(String nodeName, List<ShardRecoveryResponse> responses) {
List<ShardRecoveryResponse> nodeResponses = new ArrayList<>();
for (ShardRecoveryResponse response : responses) {
if (response.recoveryState().getTargetNode().getName().equals(nodeName)) {
nodeResponses.add(response);
private List<RecoveryState> findRecoveriesForTargetNode(String nodeName, List<RecoveryState> recoveryStates) {
List<RecoveryState> nodeResponses = new ArrayList<>();
for (RecoveryState recoveryState : recoveryStates) {
if (recoveryState.getTargetNode().getName().equals(nodeName)) {
nodeResponses.add(recoveryState);
}
}
return nodeResponses;
@ -519,7 +534,6 @@ public class IndexRecoveryIT extends ESIntegTestCase {
final Settings nodeSettings = Settings.builder()
.put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK, "100ms")
.put(RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT, "1s")
.put("plugin.types", MockTransportService.TestPlugin.class.getName())
.put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE, false) // restarted recoveries will delete temp files and write them again
.build();
// start a master node

8
core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoverySource;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.InternalTestCluster;
@ -63,6 +64,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
@ -87,10 +89,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
// which is between 1 and 2 sec can cause each of the shard deletion requests to timeout.
// to prevent this we are setting the timeout here to something highish ie. the default in practice
.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(30, TimeUnit.SECONDS))
.extendArray("plugin.types", MockTransportService.TestPlugin.class.getName())
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
@Override
protected void ensureClusterStateConsistency() throws IOException {
// testShardActiveElseWhere might change the state of a non-master node

9
core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java
View File

@ -33,6 +33,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.junit.Test;
import java.util.Collection;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
@ -41,11 +43,8 @@ import static org.hamcrest.core.IsNull.notNullValue;
public class IndexTemplateFilteringIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", TestPlugin.class.getName())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(TestPlugin.class);
}
@Test

8
core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.indices.warmer;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.segments.IndexSegments;
import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
@ -49,7 +48,10 @@ import org.junit.Test;
import java.util.Locale;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
public class SimpleIndicesWarmerIT extends ESIntegTestCase {
@ -272,7 +274,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase {
for (IndexShardSegments indexShardSegments : indicesSegments) {
for (ShardSegments shardSegments : indexShardSegments) {
for (Segment segment : shardSegments) {
logger.debug("+=" + segment.memoryInBytes + " " + indexShardSegments.getShardId() + " " + shardSegments.getIndex());
logger.debug("+=" + segment.memoryInBytes + " " + indexShardSegments.getShardId() + " " + shardSegments.getShardRouting().getIndex());
total += segment.memoryInBytes;
}
}

54
core/src/test/java/org/elasticsearch/node/MockNode.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.node;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import java.util.Collection;
/**
* A node for testing which allows:
* <ul>
* <li>Overriding Version.CURRENT</li>
* <li>Adding test plugins that exist on the classpath</li>
* </ul>
*/
public class MockNode extends Node {
// these are kept here so a copy of this MockNode can be created, since Node does not store them
private Version version;
private Collection<Class<? extends Plugin>> plugins;
public MockNode(Settings settings, boolean loadConfigSettings, Version version, Collection<Class<? extends Plugin>> classpathPlugins) {
super(settings, loadConfigSettings, version, classpathPlugins);
this.version = version;
this.plugins = classpathPlugins;
}
public Collection<Class<? extends Plugin>> getPlugins() {
return plugins;
}
public Version getVersion() {
return version;
}
}

4
core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
View File

@ -22,7 +22,7 @@ package org.elasticsearch.nodesinfo;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.plugins.PluginTestCase;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.Test;
@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.*;
*
*/
@ClusterScope(scope= Scope.TEST, numDataNodes =0)
public class SimpleNodesInfoIT extends PluginTestCase {
public class SimpleNodesInfoIT extends ESIntegTestCase {
static final class Fields {
static final String SITE_PLUGIN = "dummy";

14
core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.transport.*;
import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@ -51,16 +52,17 @@ public class PluggableTransportModuleIT extends ESIntegTestCase {
return settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put(DiscoveryModule.DISCOVERY_TYPE_KEY, "local")
.put("plugin.types", CountingSentRequestsPlugin.class.getName())
.build();
}
@Override
protected Settings transportClientSettings() {
return settingsBuilder()
.put("plugin.types", CountingSentRequestsPlugin.class.getName())
.put(super.transportClientSettings())
.build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(CountingSentRequestsPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return pluginList(CountingSentRequestsPlugin.class);
}
@Test

8
core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java
View File

@ -82,12 +82,12 @@ public class PluginManagerUnitTests extends ESTestCase {
Iterator<URL> iterator = handle.urls().iterator();
if (supportStagingUrls) {
String expectedStagingURL = String.format(Locale.ROOT, "http://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
String expectedStagingURL = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
Version.CURRENT.number(), Build.CURRENT.hashShort(), pluginName, Version.CURRENT.number(), pluginName, Version.CURRENT.number());
assertThat(iterator.next().toExternalForm(), is(expectedStagingURL));
}
URL expected = new URL("http", "download.elastic.co", "/elasticsearch/release/org/elasticsearch/plugin/" + pluginName + "/" + Version.CURRENT.number() + "/" +
URL expected = new URL("https", "download.elastic.co", "/elasticsearch/release/org/elasticsearch/plugin/" + pluginName + "/" + Version.CURRENT.number() + "/" +
pluginName + "-" + Version.CURRENT.number() + ".zip");
assertThat(iterator.next().toExternalForm(), is(expected.toExternalForm()));
@ -108,12 +108,12 @@ public class PluginManagerUnitTests extends ESTestCase {
Iterator<URL> iterator = handle.urls().iterator();
if (supportStagingUrls) {
String expectedStagingUrl = String.format(Locale.ROOT, "http://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
String expectedStagingUrl = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
Version.CURRENT.number(), Build.CURRENT.hashShort(), randomPluginName, Version.CURRENT.number(), randomPluginName, Version.CURRENT.number());
assertThat(iterator.next().toExternalForm(), is(expectedStagingUrl));
}
String releaseUrl = String.format(Locale.ROOT, "http://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
String releaseUrl = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%s/%s/%s-%s.zip",
randomPluginName, Version.CURRENT.number(), randomPluginName, Version.CURRENT.number());
assertThat(iterator.next().toExternalForm(), is(releaseUrl));

56
core/src/test/java/org/elasticsearch/plugins/PluginTestCase.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESIntegTestCase;
import java.net.URISyntaxException;
import java.net.URL;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
/**
* Base class that lets you start a node with plugins.
*/
public abstract class PluginTestCase extends ESIntegTestCase {
public String startNodeWithPlugins(Settings nodeSettings, String pluginDir, String ... pluginClassNames) throws URISyntaxException {
URL resource = getClass().getResource(pluginDir);
Settings.Builder settings = settingsBuilder();
settings.put(nodeSettings);
if (resource != null) {
settings.put("path.plugins", getDataPath(pluginDir).toAbsolutePath());
}
if (pluginClassNames.length > 0) {
settings.putArray("plugin.types", pluginClassNames);
}
String nodeName = internalCluster().startNode(settings);
// We wait for a Green status
client().admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
return internalCluster().getInstance(ClusterService.class, nodeName).state().nodes().localNodeId();
}
}

16
core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
View File

@ -24,6 +24,8 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
public class PluginsServiceTests extends ESTestCase {
public static class AdditionalSettingsPlugin1 extends Plugin {
@Override
@ -54,13 +56,16 @@ public class PluginsServiceTests extends ESTestCase {
}
}
static PluginsService newPluginsService(Settings settings, Class<? extends Plugin>... classpathPlugins) {
return new PluginsService(settings, new Environment(settings), Arrays.asList(classpathPlugins));
}
public void testAdditionalSettings() {
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.put("my.setting", "test")
.put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.SIMPLEFS.getSettingsKey())
.putArray("plugin.types", AdditionalSettingsPlugin1.class.getName()).build();
PluginsService service = new PluginsService(settings, new Environment(settings));
.put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.SIMPLEFS.getSettingsKey()).build();
PluginsService service = newPluginsService(settings, AdditionalSettingsPlugin1.class);
Settings newSettings = service.updatedSettings();
assertEquals("test", newSettings.get("my.setting")); // previous settings still exist
assertEquals("1", newSettings.get("foo.bar")); // added setting exists
@ -69,9 +74,8 @@ public class PluginsServiceTests extends ESTestCase {
public void testAdditionalSettingsClash() {
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.putArray("plugin.types", AdditionalSettingsPlugin1.class.getName(), AdditionalSettingsPlugin2.class.getName()).build();
PluginsService service = new PluginsService(settings, new Environment(settings));
.put("path.home", createTempDir()).build();
PluginsService service = newPluginsService(settings, AdditionalSettingsPlugin1.class, AdditionalSettingsPlugin2.class);
try {
service.updatedSettings();
fail("Expected exception when building updated settings");

8
core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
View File

@ -25,6 +25,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.rest.client.http.HttpResponse;
import org.junit.Test;
import java.util.Collection;
import static org.elasticsearch.rest.RestStatus.OK;
import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED;
import static org.elasticsearch.test.ESIntegTestCase.Scope;
@ -41,11 +43,15 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", TestResponseHeaderPlugin.class.getName())
.put("force.http.enabled", true)
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(TestResponseHeaderPlugin.class);
}
@Test
public void testThatSettingHeadersWorks() throws Exception {
ensureGreen();

2
core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
View File

@ -284,7 +284,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().get();
for (ShardStats shardStats : indicesStatsResponse.getShards()) {
DocsStats docsStats = shardStats.getStats().docs;
logger.info("shard [{}] - count {}, primary {}", shardStats.getShardId(), docsStats.getCount(), shardStats.getShardRouting().primary());
logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
}
//if there was an error we try to wait and see if at some point it'll get fixed

11
core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
View File

@ -54,6 +54,7 @@ import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.recovery.RecoveryTarget;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.test.BackgroundIndexer;
@ -63,7 +64,6 @@ import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestOptions;
import org.elasticsearch.transport.TransportService;
@ -76,6 +76,7 @@ import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
@ -100,12 +101,10 @@ public class RelocationIT extends ESIntegTestCase {
private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES);
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put("plugin.types", MockTransportService.TestPlugin.class.getName()).build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
@Test
public void testSimpleRelocationNoIndexing() {
logger.info("--> starting [node1] ...");
@ -422,7 +421,7 @@ public class RelocationIT extends ESIntegTestCase {
public boolean apply(Object input) {
RecoveryResponse recoveryResponse = internalCluster().client(redNodeName).admin().indices().prepareRecoveries(indexName)
.get();
return !recoveryResponse.shardResponses().get(indexName).isEmpty();
return !recoveryResponse.shardRecoveryStates().get(indexName).isEmpty();
}
}
);

9
core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.recovery.RecoveryTarget;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.*;
@ -41,6 +42,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
@ -55,14 +57,19 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@SuppressCodecs("*") // test relies on exact file extensions
public class TruncatedRecoveryIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.extendArray("plugin.types", MockTransportService.TestPlugin.class.getName())
.put(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES));
return builder.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(MockTransportService.TestPlugin.class);
}
/**
* This test tries to truncate some of larger files in the index to trigger leftovers on the recovery
* target. This happens during recovery when the last chunk of the file is transferred to the replica

8
core/src/test/java/org/elasticsearch/script/CustomScriptContextIT.java
View File

@ -28,6 +28,8 @@ import org.elasticsearch.script.mustache.MustacheScriptEngineService;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Collection;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.notNullValue;
@ -40,12 +42,16 @@ public class CustomScriptContextIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", CustomScriptContextPlugin.class.getName())
.put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "off")
.put("script.engine.expression.inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "off")
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(CustomScriptContextPlugin.class);
}
@Test
public void testCustomScriptContextsSettings() {
ScriptService scriptService = internalCluster().getInstance(ScriptService.class);

5
core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ExecutionException;
@ -40,8 +41,8 @@ import static org.hamcrest.Matchers.equalTo;
public class ScriptFieldIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put("plugin.types", CustomScriptPlugin.class.getName()).build();
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(CustomScriptPlugin.class);
}
static int[] intArray = { Integer.MAX_VALUE, Integer.MIN_VALUE, 3 };

6
core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
View File

@ -93,11 +93,15 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
public Settings nodeSettings(int nodeOrdinal) {
return settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", CustomSignificanceHeuristicPlugin.class.getName())
.put("path.conf", this.getDataPath("config"))
.build();
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(CustomSignificanceHeuristicPlugin.class);
}
public String randomExecutionHint() {
return randomBoolean() ? null : randomFrom(SignificantTermsAggregatorFactory.ExecutionMode.values()).toString();
}

Some files were not shown because too many files have changed in this diff Show More