Add `prefer_local` flag to analyze and percolate request, closes #625.

This commit is contained in:
kimchy 2011-01-14 01:01:37 +02:00
parent ff347858c5
commit 38d10d19bc
12 changed files with 143 additions and 37 deletions

View File

@ -79,6 +79,15 @@ public class AnalyzeRequest extends SingleCustomOperationRequest {
return this.analyzer; return this.analyzer;
} }
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
@Override public AnalyzeRequest preferLocal(boolean preferLocal) {
super.preferLocal(preferLocal);
return this;
}
@Override public ActionRequestValidationException validate() { @Override public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = super.validate(); ActionRequestValidationException validationException = super.validate();
if (index == null) { if (index == null) {

View File

@ -75,7 +75,7 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction
@Override protected ShardsIterator shards(ClusterState clusterState, AnalyzeRequest request) { @Override protected ShardsIterator shards(ClusterState clusterState, AnalyzeRequest request) {
request.index(clusterState.metaData().concreteIndex(request.index())); request.index(clusterState.metaData().concreteIndex(request.index()));
return clusterState.routingTable().index(request.index()).allShardsIt(); return clusterState.routingTable().index(request.index()).randomAllShardsIt();
} }
@Override protected AnalyzeResponse shardOperation(AnalyzeRequest request, int shardId) throws ElasticSearchException { @Override protected AnalyzeResponse shardOperation(AnalyzeRequest request, int shardId) throws ElasticSearchException {

View File

@ -150,6 +150,15 @@ public class PercolateRequest extends SingleCustomOperationRequest {
return this; return this;
} }
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
@Override public PercolateRequest preferLocal(boolean preferLocal) {
super.preferLocal(preferLocal);
return this;
}
@Override public ActionRequestValidationException validate() { @Override public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = super.validate(); ActionRequestValidationException validationException = super.validate();
if (index == null) { if (index == null) {

View File

@ -65,7 +65,7 @@ public class TransportPercolateAction extends TransportSingleCustomOperationActi
@Override protected ShardsIterator shards(ClusterState clusterState, PercolateRequest request) { @Override protected ShardsIterator shards(ClusterState clusterState, PercolateRequest request) {
request.index(clusterState.metaData().concreteIndex(request.index())); request.index(clusterState.metaData().concreteIndex(request.index()));
return clusterState.routingTable().index(request.index()).allShardsIt(); return clusterState.routingTable().index(request.index()).randomAllShardsIt();
} }
@Override protected PercolateResponse shardOperation(PercolateRequest request, int shardId) throws ElasticSearchException { @Override protected PercolateResponse shardOperation(PercolateRequest request, int shardId) throws ElasticSearchException {

View File

@ -33,6 +33,7 @@ public abstract class SingleCustomOperationRequest implements ActionRequest {
private boolean threadedListener = false; private boolean threadedListener = false;
private boolean threadedOperation = true; private boolean threadedOperation = true;
private boolean preferLocal = true;
protected SingleCustomOperationRequest() { protected SingleCustomOperationRequest() {
} }
@ -68,15 +69,34 @@ public abstract class SingleCustomOperationRequest implements ActionRequest {
return this; return this;
} }
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
public SingleCustomOperationRequest preferLocal(boolean preferLocal) {
this.preferLocal = preferLocal;
return this;
}
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
public boolean preferLocalShard() {
return this.preferLocal;
}
public void beforeLocalFork() { public void beforeLocalFork() {
} }
@Override public void readFrom(StreamInput in) throws IOException { @Override public void readFrom(StreamInput in) throws IOException {
// no need to pass threading over the network, they are always false when coming throw a thread pool // no need to pass threading over the network, they are always false when coming throw a thread pool
preferLocal = in.readBoolean();
} }
@Override public void writeTo(StreamOutput out) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(preferLocal);
} }
} }

View File

@ -116,40 +116,44 @@ public abstract class TransportSingleCustomOperationAction<Request extends Singl
* First get should try and use a shard that exists on a local node for better performance * First get should try and use a shard that exists on a local node for better performance
*/ */
private void performFirst() { private void performFirst() {
while (shardsIt.hasNextActive()) { if (request.preferLocalShard()) {
final ShardRouting shard = shardsIt.nextActive(); while (shardsIt.hasNextActive()) {
if (shard.currentNodeId().equals(nodes.localNodeId())) { final ShardRouting shard = shardsIt.nextActive();
if (request.operationThreaded()) { if (shard.currentNodeId().equals(nodes.localNodeId())) {
request.beforeLocalFork(); if (request.operationThreaded()) {
threadPool.execute(new Runnable() { request.beforeLocalFork();
@Override public void run() { threadPool.execute(new Runnable() {
try { @Override public void run() {
Response response = shardOperation(request, shard.id()); try {
listener.onResponse(response); Response response = shardOperation(request, shard.id());
} catch (Exception e) {
onFailure(shard, e);
}
}
});
return;
} else {
try {
final Response response = shardOperation(request, shard.id());
if (request.listenerThreaded()) {
threadPool.execute(new Runnable() {
@Override public void run() {
listener.onResponse(response); listener.onResponse(response);
} catch (Exception e) {
onFailure(shard, e);
} }
}); }
} else { });
listener.onResponse(response);
}
return; return;
} catch (Exception e) { } else {
onFailure(shard, e); try {
final Response response = shardOperation(request, shard.id());
if (request.listenerThreaded()) {
threadPool.execute(new Runnable() {
@Override public void run() {
listener.onResponse(response);
}
});
} else {
listener.onResponse(response);
}
return;
} catch (Exception e) {
onFailure(shard, e);
}
} }
} }
} }
} else {
perform(null);
} }
if (!shardsIt.hasNextActive()) { if (!shardsIt.hasNextActive()) {
// no local node get, go remote // no local node get, go remote
@ -162,7 +166,41 @@ public abstract class TransportSingleCustomOperationAction<Request extends Singl
while (shardsIt.hasNextActive()) { while (shardsIt.hasNextActive()) {
final ShardRouting shard = shardsIt.nextActive(); final ShardRouting shard = shardsIt.nextActive();
// no need to check for local nodes, we tried them already in performFirstGet // no need to check for local nodes, we tried them already in performFirstGet
if (!shard.currentNodeId().equals(nodes.localNodeId())) { if (shard.currentNodeId().equals(nodes.localNodeId())) {
// we don't prefer local shard, so try and do it here
if (!request.preferLocalShard()) {
if (request.operationThreaded()) {
request.beforeLocalFork();
threadPool.execute(new Runnable() {
@Override public void run() {
try {
Response response = shardOperation(request, shard.id());
listener.onResponse(response);
} catch (Exception e) {
onFailure(shard, e);
}
}
});
return;
} else {
try {
final Response response = shardOperation(request, shard.id());
if (request.listenerThreaded()) {
threadPool.execute(new Runnable() {
@Override public void run() {
listener.onResponse(response);
}
});
} else {
listener.onResponse(response);
}
return;
} catch (Exception e) {
onFailure(shard, e);
}
}
}
} else {
DiscoveryNode node = nodes.get(shard.currentNodeId()); DiscoveryNode node = nodes.get(shard.currentNodeId());
transportService.sendRequest(node, transportShardAction(), new ShardSingleOperationRequest(request, shard.id()), new BaseTransportResponseHandler<Response>() { transportService.sendRequest(node, transportShardAction(), new ShardSingleOperationRequest(request, shard.id()), new BaseTransportResponseHandler<Response>() {
@Override public Response newInstance() { @Override public Response newInstance() {

View File

@ -44,6 +44,15 @@ public class AnalyzeRequestBuilder extends BaseIndicesRequestBuilder<AnalyzeRequ
return this; return this;
} }
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
public AnalyzeRequestBuilder setPreferLocal(boolean preferLocal) {
request.preferLocal(preferLocal);
return this;
}
@Override protected void doExecute(ActionListener<AnalyzeResponse> listener) { @Override protected void doExecute(ActionListener<AnalyzeResponse> listener) {
client.analyze(request, listener); client.analyze(request, listener);
} }

View File

@ -128,6 +128,15 @@ public class PercolateRequestBuilder extends BaseRequestBuilder<PercolateRequest
return this; return this;
} }
/**
* if this operation hits a node with a local relevant shard, should it be preferred
* to be executed on, or just do plain round robin. Defaults to <tt>true</tt>
*/
public PercolateRequestBuilder setPreferLocal(boolean preferLocal) {
request.preferLocal(preferLocal);
return this;
}
/** /**
* Controls if the operation will be executed on a separate thread when executed locally. Defaults * Controls if the operation will be executed on a separate thread when executed locally. Defaults
* to <tt>true</tt> when running in embedded mode. * to <tt>true</tt> when running in embedded mode.

View File

@ -153,7 +153,7 @@ public class IndexRoutingTable implements Iterable<IndexShardRoutingTable> {
/** /**
* An iterator over all shards (including replicas). * An iterator over all shards (including replicas).
*/ */
public ShardsIterator allShardsIt() { public ShardsIterator randomAllShardsIt() {
return new PlainShardsIterator(allShards, Math.abs(counter.incrementAndGet())); return new PlainShardsIterator(allShards, Math.abs(counter.incrementAndGet()));
} }

View File

@ -61,6 +61,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
} }
AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index"), text); AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index"), text);
analyzeRequest.preferLocal(request.paramAsBoolean("prefer_local", analyzeRequest.preferLocalShard()));
analyzeRequest.analyzer(request.param("analyzer")); analyzeRequest.analyzer(request.param("analyzer"));
client.admin().indices().analyze(analyzeRequest, new ActionListener<AnalyzeResponse>() { client.admin().indices().analyze(analyzeRequest, new ActionListener<AnalyzeResponse>() {
@Override public void onResponse(AnalyzeResponse response) { @Override public void onResponse(AnalyzeResponse response) {

View File

@ -55,6 +55,7 @@ public class RestPercolateAction extends BaseRestHandler {
// we don't spawn, then fork if local // we don't spawn, then fork if local
percolateRequest.operationThreaded(true); percolateRequest.operationThreaded(true);
percolateRequest.preferLocal(request.paramAsBoolean("prefer_local", percolateRequest.preferLocalShard()));
client.percolate(percolateRequest, new ActionListener<PercolateResponse>() { client.percolate(percolateRequest, new ActionListener<PercolateResponse>() {
@Override public void onResponse(PercolateResponse response) { @Override public void onResponse(PercolateResponse response) {
try { try {

View File

@ -101,11 +101,21 @@ public class SimplePercolatorTests extends AbstractNodesTests {
.execute().actionGet(); .execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForActiveShards(4).execute().actionGet(); client.admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForActiveShards(4).execute().actionGet();
PercolateResponse percolate = client.preparePercolate("test").setSource(jsonBuilder().startObject().startObject("doc").startObject("type1") for (int i = 0; i < 10; i++) {
.field("field1", "value1") PercolateResponse percolate = client.preparePercolate("test").setSource(jsonBuilder().startObject().startObject("doc").startObject("type1")
.endObject().endObject().endObject()) .field("field1", "value1")
.execute().actionGet(); .endObject().endObject().endObject())
assertThat(percolate.matches().size(), equalTo(1)); .execute().actionGet();
assertThat(percolate.matches().size(), equalTo(1));
}
for (int i = 0; i < 10; i++) {
PercolateResponse percolate = client.preparePercolate("test").setPreferLocal(false).setSource(jsonBuilder().startObject().startObject("doc").startObject("type1")
.field("field1", "value1")
.endObject().endObject().endObject())
.execute().actionGet();
assertThat(percolate.matches().size(), equalTo(1));
}
} }
@Test public void dynamicAddingRemovingQueries() throws Exception { @Test public void dynamicAddingRemovingQueries() throws Exception {