iterator() {
- return Iterators.forArray(responses);
+ return Arrays.stream(responses).iterator();
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index c171ae9af1a..ad7b9c11763 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -49,14 +49,14 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
/**
* Index request to index a typed JSON document into a specific index and make it searchable. Best
* created using {@link org.elasticsearch.client.Requests#indexRequest(String)}.
- *
+ *
* The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and
* {@link #source(byte[])} to be set.
- *
+ *
* The source (content to index) can be set in its bytes form using ({@link #source(byte[])}),
* its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder}
* ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}).
- *
+ *
* If the {@link #id(String)} is not set, it will be automatically generated.
*
* @see IndexResponse
@@ -114,7 +114,7 @@ public class IndexRequest extends ReplicationRequest implements Do
public static OpType fromString(String sOpType) {
String lowersOpType = sOpType.toLowerCase(Locale.ROOT);
- switch(lowersOpType){
+ switch (lowersOpType) {
case "create":
return OpType.CREATE;
case "index":
@@ -216,6 +216,14 @@ public class IndexRequest extends ReplicationRequest implements Do
if (source == null) {
validationException = addValidationError("source is missing", validationException);
}
+
+ if (opType() == OpType.CREATE) {
+ if (versionType != VersionType.INTERNAL || version != Versions.MATCH_DELETED) {
+ validationException = addValidationError("create operations do not support versioning. use index instead", validationException);
+ return validationException;
+ }
+ }
+
if (!versionType.validateVersionForWrites(version)) {
validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
}
@@ -370,7 +378,7 @@ public class IndexRequest extends ReplicationRequest implements Do
/**
* Sets the document source to index.
- *
+ *
* Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}
* or using the {@link #source(byte[])}.
*/
@@ -480,6 +488,10 @@ public class IndexRequest extends ReplicationRequest implements Do
*/
public IndexRequest opType(OpType opType) {
this.opType = opType;
+ if (opType == OpType.CREATE) {
+ version(Versions.MATCH_DELETED);
+ versionType(VersionType.INTERNAL);
+ }
return this;
}
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index 3e98f1a32c2..63b82377d8a 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -54,7 +54,7 @@ import org.elasticsearch.transport.TransportService;
/**
* Performs the index operation.
- *
+ *
* Allows for the following settings:
*
* autoCreateIndex : When set to true , will automatically create an index if one does not exists.
@@ -167,6 +167,7 @@ public class TransportIndexAction extends TransportReplicationAction result = executeIndexRequestOnPrimary(null, request, indexShard);
+
final IndexResponse response = result.response;
final Translog.Location location = result.location;
processAfter(request.refresh(), indexShard, location);
@@ -180,18 +181,12 @@ public class TransportIndexAction extends TransportReplicationAction iterator() {
- return Iterators.forArray(items);
+ return Arrays.stream(items).iterator();
}
/**
diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
index ffe476366f7..3540daa255c 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
@@ -19,12 +19,12 @@
package org.elasticsearch.action.search;
-import org.elasticsearch.Version;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.StatusToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
@@ -69,6 +69,8 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(Fields.SUCCEEDED, succeeded);
+ builder.field(Fields.NUMFREED, numFreed);
return builder;
}
@@ -85,4 +87,10 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
out.writeBoolean(succeeded);
out.writeVInt(numFreed);
}
+
+ static final class Fields {
+ static final XContentBuilderString SUCCEEDED = new XContentBuilderString("succeeded");
+ static final XContentBuilderString NUMFREED = new XContentBuilderString("num_freed");
+ }
+
}
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index 80745652be3..0a9d6191030 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -19,7 +19,6 @@
package org.elasticsearch.action.search;
-import com.google.common.collect.Iterators;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
@@ -32,7 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.common.xcontent.XContentFactory;
import java.io.IOException;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.Iterator;
/**
@@ -122,7 +121,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable iterator() {
- return Iterators.forArray(items);
+ return Arrays.stream(items).iterator();
}
/**
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 118e1124c33..0597c26f636 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
@@ -95,17 +96,22 @@ public abstract class TransportNodesAction listener;
- private final ClusterState clusterState;
private final AtomicReferenceArray responses;
private final AtomicInteger counter = new AtomicInteger();
private AsyncAction(NodesRequest request, ActionListener listener) {
this.request = request;
this.listener = listener;
- clusterState = clusterService.state();
+ ClusterState clusterState = clusterService.state();
String[] nodesIds = resolveNodes(request, clusterState);
this.nodesIds = filterNodeIds(clusterState.nodes(), nodesIds);
+ ImmutableOpenMap nodes = clusterState.nodes().nodes();
+ this.nodes = new DiscoveryNode[nodesIds.length];
+ for (int i = 0; i < nodesIds.length; i++) {
+ this.nodes[i] = nodes.get(nodesIds[i]);
+ }
this.responses = new AtomicReferenceArray<>(this.nodesIds.length);
}
@@ -128,7 +134,7 @@ public abstract class TransportNodesAction executeIndexRequestOnPrimary(BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) throws Throwable {
- Engine.IndexingOperation operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard);
+ Engine.Index operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard);
Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
final ShardId shardId = indexShard.shardId();
if (update != null) {
@@ -1064,7 +1055,7 @@ public abstract class TransportReplicationAction listener) {
+ AsyncSingleAction(Request request, ActionListener listener) {
this.request = request;
this.listener = listener;
}
@@ -123,14 +125,14 @@ public abstract class TransportInstanceSingleOperationAction iterator() {
- return Iterators.unmodifiableIterator(requests.iterator());
+ return Collections.unmodifiableCollection(requests).iterator();
}
public boolean isEmpty() {
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index fe013d540d2..6eb3b3277e6 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -19,7 +19,6 @@
package org.elasticsearch.action.termvectors;
-import com.google.common.collect.Iterators;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.io.stream.StreamInput;
@@ -30,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Iterator;
public class MultiTermVectorsResponse extends ActionResponse implements Iterable, ToXContent {
@@ -120,7 +120,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
@Override
public Iterator iterator() {
- return Iterators.forArray(responses);
+ return Arrays.stream(responses).iterator();
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 7479416b122..2a639c83ad1 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -48,9 +48,8 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.engine.DocumentAlreadyExistsException;
-import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndicesService;
@@ -170,7 +169,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
final UpdateHelper.Result result = updateHelper.prepare(request, indexShard);
switch (result.operation()) {
case UPSERT:
- IndexRequest upsertRequest = new IndexRequest((IndexRequest)result.action(), request);
+ IndexRequest upsertRequest = new IndexRequest(result.action(), request);
// we fetch it from the index request so we don't generate the bytes twice, its already done in the index request
final BytesReference upsertSourceBytes = upsertRequest.source();
indexAction.execute(upsertRequest, new ActionListener() {
@@ -189,7 +188,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
@Override
public void onFailure(Throwable e) {
e = ExceptionsHelper.unwrapCause(e);
- if (e instanceof VersionConflictEngineException || e instanceof DocumentAlreadyExistsException) {
+ if (e instanceof VersionConflictEngineException) {
if (retryCount < request.retryOnConflict()) {
threadPool.executor(executor()).execute(new ActionRunnable(listener) {
@Override
@@ -205,7 +204,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
});
break;
case INDEX:
- IndexRequest indexRequest = new IndexRequest((IndexRequest)result.action(), request);
+ IndexRequest indexRequest = new IndexRequest(result.action(), request);
// we fetch it from the index request so we don't generate the bytes twice, its already done in the index request
final BytesReference indexSourceBytes = indexRequest.source();
indexAction.execute(indexRequest, new ActionListener() {
@@ -235,7 +234,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
});
break;
case DELETE:
- DeleteRequest deleteRequest = new DeleteRequest((DeleteRequest)result.action(), request);
+ DeleteRequest deleteRequest = new DeleteRequest(result.action(), request);
deleteAction.execute(deleteRequest, new ActionListener() {
@Override
public void onResponse(DeleteResponse response) {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 542444b4097..9ebb2c97627 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.PidFile;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.cli.CliTool;
import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.CreationException;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.ESLogger;
@@ -249,13 +248,13 @@ final class Bootstrap {
Environment environment = initialSettings(foreground);
Settings settings = environment.settings();
+ setupLogging(settings, environment);
+ checkForCustomConfFile();
if (environment.pidFile() != null) {
PidFile.create(environment.pidFile(), true);
}
- setupLogging(settings, environment);
-
if (System.getProperty("es.max-open-files", "false").equals("true")) {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
logger.info("max_open_files [{}]", ProcessProbe.getInstance().getMaxFileDescriptorCount());
@@ -330,4 +329,21 @@ final class Bootstrap {
System.err.flush();
}
}
+
+ private static void checkForCustomConfFile() {
+ String confFileSetting = System.getProperty("es.default.config");
+ checkUnsetAndMaybeExit(confFileSetting, "es.default.config");
+ confFileSetting = System.getProperty("es.config");
+ checkUnsetAndMaybeExit(confFileSetting, "es.config");
+ confFileSetting = System.getProperty("elasticsearch.config");
+ checkUnsetAndMaybeExit(confFileSetting, "elasticsearch.config");
+ }
+
+ private static void checkUnsetAndMaybeExit(String confFileSetting, String settingName) {
+ if (confFileSetting != null && confFileSetting.isEmpty() == false) {
+ ESLogger logger = Loggers.getLogger(Bootstrap.class);
+ logger.info("{} is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.", settingName);
+ System.exit(1);
+ }
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index b27048d22bd..66dda6ee774 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -165,7 +165,7 @@ final class Security {
Map m = new HashMap<>();
m.put("repository-s3", "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin");
m.put("discovery-ec2", "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin");
- m.put("cloud-gce", "org.elasticsearch.plugin.cloud.gce.CloudGcePlugin");
+ m.put("discovery-gce", "org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin");
m.put("lang-expression", "org.elasticsearch.script.expression.ExpressionPlugin");
m.put("lang-groovy", "org.elasticsearch.script.groovy.GroovyPlugin");
m.put("lang-javascript", "org.elasticsearch.plugin.javascript.JavaScriptPlugin");
diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java
index eafac2b6509..6e0b0b27fd9 100644
--- a/core/src/main/java/org/elasticsearch/client/Client.java
+++ b/core/src/main/java/org/elasticsearch/client/Client.java
@@ -21,9 +21,6 @@ package org.elasticsearch.client;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequestBuilder;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index c3eb51585c2..1be22b257e7 100644
--- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -77,6 +77,9 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
/**
* Administrative actions/operations against indices.
@@ -423,4 +426,25 @@ public interface ClusterAdminClient extends ElasticsearchClient {
*/
SnapshotsStatusRequestBuilder prepareSnapshotStatus();
+
+ /**
+ * Return the rendered search request for a given search template.
+ *
+ * @param request The request
+ * @return The result future
+ */
+ ActionFuture renderSearchTemplate(RenderSearchTemplateRequest request);
+
+ /**
+ * Return the rendered search request for a given search template.
+ *
+ * @param request The request
+ * @param listener A listener to be notified of the result
+ */
+ void renderSearchTemplate(RenderSearchTemplateRequest request, ActionListener listener);
+
+ /**
+ * Return the rendered search request for a given search template.
+ */
+ RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate();
}
diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index 755bf333e59..75cae17ea9d 100644
--- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -105,9 +105,6 @@ import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequestBuilder;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateResponse;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerResponse;
@@ -746,27 +743,6 @@ public interface IndicesAdminClient extends ElasticsearchClient {
*/
ValidateQueryRequestBuilder prepareValidateQuery(String... indices);
- /**
- * Return the rendered search request for a given search template.
- *
- * @param request The request
- * @return The result future
- */
- ActionFuture renderSearchTemplate(RenderSearchTemplateRequest request);
-
- /**
- * Return the rendered search request for a given search template.
- *
- * @param request The request
- * @param listener A listener to be notified of the result
- */
- void renderSearchTemplate(RenderSearchTemplateRequest request, ActionListener listener);
-
- /**
- * Return the rendered search request for a given search template.
- */
- RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate();
-
/**
* Puts an index search warmer to be applies when applicable.
*/
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index f9abf2f0437..3fa5d789905 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -208,10 +208,10 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateAction;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateRequestBuilder;
-import org.elasticsearch.action.admin.indices.validate.template.RenderSearchTemplateResponse;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder;
+import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerAction;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest;
import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder;
@@ -1142,6 +1142,21 @@ public abstract class AbstractClient extends AbstractComponent implements Client
public SnapshotsStatusRequestBuilder prepareSnapshotStatus() {
return new SnapshotsStatusRequestBuilder(this, SnapshotsStatusAction.INSTANCE);
}
+
+ @Override
+ public ActionFuture renderSearchTemplate(final RenderSearchTemplateRequest request) {
+ return execute(RenderSearchTemplateAction.INSTANCE, request);
+ }
+
+ @Override
+ public void renderSearchTemplate(final RenderSearchTemplateRequest request, final ActionListener listener) {
+ execute(RenderSearchTemplateAction.INSTANCE, request, listener);
+ }
+
+ @Override
+ public RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate() {
+ return new RenderSearchTemplateRequestBuilder(this, RenderSearchTemplateAction.INSTANCE);
+ }
}
static class IndicesAdmin implements IndicesAdminClient {
@@ -1617,21 +1632,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client
return new ValidateQueryRequestBuilder(this, ValidateQueryAction.INSTANCE).setIndices(indices);
}
- @Override
- public ActionFuture renderSearchTemplate(final RenderSearchTemplateRequest request) {
- return execute(RenderSearchTemplateAction.INSTANCE, request);
- }
-
- @Override
- public void renderSearchTemplate(final RenderSearchTemplateRequest request, final ActionListener listener) {
- execute(RenderSearchTemplateAction.INSTANCE, request, listener);
- }
-
- @Override
- public RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate() {
- return new RenderSearchTemplateRequestBuilder(this, RenderSearchTemplateAction.INSTANCE);
- }
-
@Override
public ActionFuture putWarmer(PutWarmerRequest request) {
return execute(PutWarmerAction.INSTANCE, request);
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
index b13c7991b5e..e3925aa6f4e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
@@ -73,7 +73,7 @@ public class MappingUpdatedAction extends AbstractComponent {
throw new IllegalArgumentException("_default_ mapping should not be updated");
}
return client.preparePutMapping(index).setType(type).setSource(mappingUpdate.toString())
- .setMasterNodeTimeout(timeout).setTimeout(timeout);
+ .setMasterNodeTimeout(timeout).setTimeout(timeout);
}
public void updateMappingOnMaster(String index, String type, Mapping mappingUpdate, final TimeValue timeout, final MappingUpdateListener listener) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 6ea1d0e6e61..9d110170f52 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.Diffable;
@@ -29,8 +28,6 @@ import org.elasticsearch.cluster.DiffableUtils;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
-import org.elasticsearch.cluster.routing.HashFunction;
-import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -167,16 +164,12 @@ public class IndexMetaData implements Diffable, FromXContentBuild
public static final String SETTING_PRIORITY = "index.priority";
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
public static final String SETTING_INDEX_UUID = "index.uuid";
- public static final String SETTING_LEGACY_ROUTING_HASH_FUNCTION = "index.legacy.routing.hash.type";
- public static final String SETTING_LEGACY_ROUTING_USE_TYPE = "index.legacy.routing.use_type";
public static final String SETTING_DATA_PATH = "index.data_path";
public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node";
public static final String INDEX_UUID_NA_VALUE = "_na_";
- // hard-coded hash function as of 2.0
- // older indices will read which hash function to use in their index settings
- private static final HashFunction MURMUR3_HASH_FUNCTION = new Murmur3HashFunction();
+
private final String index;
private final long version;
@@ -200,8 +193,6 @@ public class IndexMetaData implements Diffable, FromXContentBuild
private final Version indexCreatedVersion;
private final Version indexUpgradedVersion;
private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
- private final HashFunction routingHashFunction;
- private final boolean useTypeForRouting;
private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) {
if (settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) == null) {
@@ -249,23 +240,6 @@ public class IndexMetaData implements Diffable, FromXContentBuild
} else {
this.minimumCompatibleLuceneVersion = null;
}
- final String hashFunction = settings.get(SETTING_LEGACY_ROUTING_HASH_FUNCTION);
- if (hashFunction == null) {
- routingHashFunction = MURMUR3_HASH_FUNCTION;
- } else {
- final Class hashFunctionClass;
- try {
- hashFunctionClass = Class.forName(hashFunction).asSubclass(HashFunction.class);
- } catch (ClassNotFoundException|NoClassDefFoundError e) {
- throw new ElasticsearchException("failed to load custom hash function [" + hashFunction + "]", e);
- }
- try {
- routingHashFunction = hashFunctionClass.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
- throw new IllegalStateException("Cannot instantiate hash function", e);
- }
- }
- useTypeForRouting = settings.getAsBoolean(SETTING_LEGACY_ROUTING_USE_TYPE, false);
}
public String index() {
@@ -335,29 +309,6 @@ public class IndexMetaData implements Diffable, FromXContentBuild
return minimumCompatibleLuceneVersion;
}
- /**
- * Return the {@link HashFunction} that should be used for routing.
- */
- public HashFunction routingHashFunction() {
- return routingHashFunction;
- }
-
- public HashFunction getRoutingHashFunction() {
- return routingHashFunction();
- }
-
- /**
- * Return whether routing should use the _type in addition to the _id in
- * order to decide which shard a document should go to.
- */
- public boolean routingUseType() {
- return useTypeForRouting;
- }
-
- public boolean getRoutingUseType() {
- return routingUseType();
- }
-
public long creationDate() {
return settings.getAsLong(SETTING_CREATION_DATE, -1l);
}
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index a17fe044dcb..cdde49170d4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -21,11 +21,7 @@ package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
-import org.elasticsearch.cluster.routing.DjbHashFunction;
-import org.elasticsearch.cluster.routing.HashFunction;
-import org.elasticsearch.cluster.routing.SimpleHashFunction;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
@@ -34,8 +30,7 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
-import org.elasticsearch.index.store.IndexStoreModule;
+import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.script.ScriptService;
import java.util.Locale;
@@ -54,47 +49,12 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
*/
public class MetaDataIndexUpgradeService extends AbstractComponent {
- private static final String DEPRECATED_SETTING_ROUTING_HASH_FUNCTION = "cluster.routing.operation.hash.type";
- private static final String DEPRECATED_SETTING_ROUTING_USE_TYPE = "cluster.routing.operation.use_type";
-
- private final Class pre20HashFunction;
- private final Boolean pre20UseType;
private final ScriptService scriptService;
@Inject
public MetaDataIndexUpgradeService(Settings settings, ScriptService scriptService) {
super(settings);
this.scriptService = scriptService;
- final String pre20HashFunctionName = settings.get(DEPRECATED_SETTING_ROUTING_HASH_FUNCTION, null);
- final boolean hasCustomPre20HashFunction = pre20HashFunctionName != null;
- // the hash function package has changed we replace the two hash functions if their fully qualified name is used.
- if (hasCustomPre20HashFunction) {
- switch (pre20HashFunctionName) {
- case "Simple":
- case "simple":
- case "org.elasticsearch.cluster.routing.operation.hash.simple.SimpleHashFunction":
- pre20HashFunction = SimpleHashFunction.class;
- break;
- case "Djb":
- case "djb":
- case "org.elasticsearch.cluster.routing.operation.hash.djb.DjbHashFunction":
- pre20HashFunction = DjbHashFunction.class;
- break;
- default:
- try {
- pre20HashFunction = Class.forName(pre20HashFunctionName).asSubclass(HashFunction.class);
- } catch (ClassNotFoundException|NoClassDefFoundError e) {
- throw new ElasticsearchException("failed to load custom hash function [" + pre20HashFunctionName + "]", e);
- }
- }
- } else {
- pre20HashFunction = DjbHashFunction.class;
- }
- pre20UseType = settings.getAsBoolean(DEPRECATED_SETTING_ROUTING_USE_TYPE, null);
- if (hasCustomPre20HashFunction || pre20UseType != null) {
- logger.warn("Settings [{}] and [{}] are deprecated. Index settings from your old indices have been updated to record the fact that they "
- + "used some custom routing logic, you can now remove these settings from your `elasticsearch.yml` file", DEPRECATED_SETTING_ROUTING_HASH_FUNCTION, DEPRECATED_SETTING_ROUTING_USE_TYPE);
- }
}
/**
@@ -110,68 +70,29 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
return indexMetaData;
}
checkSupportedVersion(indexMetaData);
- IndexMetaData newMetaData = upgradeLegacyRoutingSettings(indexMetaData);
+ IndexMetaData newMetaData = indexMetaData;
newMetaData = addDefaultUnitsIfNeeded(newMetaData);
checkMappingsCompatibility(newMetaData);
- newMetaData = upgradeSettings(newMetaData);
newMetaData = markAsUpgraded(newMetaData);
return newMetaData;
}
- IndexMetaData upgradeSettings(IndexMetaData indexMetaData) {
- final String storeType = indexMetaData.getSettings().get(IndexStoreModule.STORE_TYPE);
- if (storeType != null) {
- final String upgradeStoreType;
- switch (storeType.toLowerCase(Locale.ROOT)) {
- case "nio_fs":
- case "niofs":
- upgradeStoreType = "niofs";
- break;
- case "mmap_fs":
- case "mmapfs":
- upgradeStoreType = "mmapfs";
- break;
- case "simple_fs":
- case "simplefs":
- upgradeStoreType = "simplefs";
- break;
- case "default":
- upgradeStoreType = "default";
- break;
- case "fs":
- upgradeStoreType = "fs";
- break;
- default:
- upgradeStoreType = storeType;
- }
- if (storeType.equals(upgradeStoreType) == false) {
- Settings indexSettings = Settings.builder().put(indexMetaData.settings())
- .put(IndexStoreModule.STORE_TYPE, upgradeStoreType)
- .build();
- return IndexMetaData.builder(indexMetaData)
- .version(indexMetaData.version())
- .settings(indexSettings)
- .build();
- }
- }
- return indexMetaData;
- }
/**
* Checks if the index was already opened by this version of Elasticsearch and doesn't require any additional checks.
*/
private boolean isUpgraded(IndexMetaData indexMetaData) {
- return indexMetaData.upgradeVersion().onOrAfter(Version.V_2_0_0_beta1);
+ return indexMetaData.upgradeVersion().onOrAfter(Version.V_3_0_0);
}
/**
- * Elasticsearch 2.0 no longer supports indices with pre Lucene v4.0 (Elasticsearch v 0.90.0) segments. All indices
- * that were created before Elasticsearch v0.90.0 should be upgraded using upgrade plugin before they can
+ * Elasticsearch 3.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1) segments. All indices
+ * that were created before Elasticsearch v2.0.0.beta1 should be upgraded using upgrade API before they can
* be open by this version of elasticsearch.
*/
private void checkSupportedVersion(IndexMetaData indexMetaData) {
if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData) == false) {
- throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v0.90.0 and wasn't upgraded."
+ throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1 and wasn't upgraded."
+ " This index should be open using a version before " + Version.CURRENT.minimumCompatibilityVersion()
+ " and upgraded using the upgrade API.");
}
@@ -181,44 +102,18 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
* Returns true if this index can be supported by the current version of elasticsearch
*/
private static boolean isSupportedVersion(IndexMetaData indexMetaData) {
- if (indexMetaData.creationVersion().onOrAfter(Version.V_0_90_0_Beta1)) {
- // The index was created with elasticsearch that was using Lucene 4.0
+ if (indexMetaData.creationVersion().onOrAfter(Version.V_2_0_0_beta1)) {
+ // The index was created with elasticsearch that was using Lucene 5.2.1
return true;
}
if (indexMetaData.getMinimumCompatibleVersion() != null &&
- indexMetaData.getMinimumCompatibleVersion().onOrAfter(org.apache.lucene.util.Version.LUCENE_4_0_0)) {
+ indexMetaData.getMinimumCompatibleVersion().onOrAfter(org.apache.lucene.util.Version.LUCENE_5_0_0)) {
//The index was upgraded we can work with it
return true;
}
return false;
}
- /**
- * Elasticsearch 2.0 deprecated custom routing hash functions. So what we do here is that for old indices, we
- * move this old and deprecated node setting to an index setting so that we can keep things backward compatible.
- */
- private IndexMetaData upgradeLegacyRoutingSettings(IndexMetaData indexMetaData) {
- if (indexMetaData.settings().get(IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION) == null
- && indexMetaData.getCreationVersion().before(Version.V_2_0_0_beta1)) {
- // these settings need an upgrade
- Settings indexSettings = Settings.builder().put(indexMetaData.settings())
- .put(IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION, pre20HashFunction)
- .put(IndexMetaData.SETTING_LEGACY_ROUTING_USE_TYPE, pre20UseType == null ? false : pre20UseType)
- .build();
- return IndexMetaData.builder(indexMetaData)
- .version(indexMetaData.version())
- .settings(indexSettings)
- .build();
- } else if (indexMetaData.getCreationVersion().onOrAfter(Version.V_2_0_0_beta1)) {
- if (indexMetaData.getSettings().get(IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION) != null
- || indexMetaData.getSettings().get(IndexMetaData.SETTING_LEGACY_ROUTING_USE_TYPE) != null) {
- throw new IllegalStateException("Index [" + indexMetaData.getIndex() + "] created on or after 2.0 should NOT contain [" + IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION
- + "] + or [" + IndexMetaData.SETTING_LEGACY_ROUTING_USE_TYPE + "] in its index settings");
- }
- }
- return indexMetaData;
- }
-
/** All known byte-sized settings for an index. */
public static final Set INDEX_BYTES_SIZE_SETTINGS = unmodifiableSet(newHashSet(
"index.merge.policy.floor_segment",
@@ -322,11 +217,11 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
Index index = new Index(indexMetaData.getIndex());
Settings settings = indexMetaData.settings();
try {
- SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings);
+ SimilarityService similarityService = new SimilarityService(index, settings);
// We cannot instantiate real analysis server at this point because the node might not have
// been started yet. However, we don't really need real analyzers at this stage - so we can fake it
try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) {
- try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService, scriptService)) {
+ try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityService, scriptService)) {
for (ObjectCursor cursor : indexMetaData.getMappings().values()) {
MappingMetaData mappingMetaData = cursor.value;
mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java
index 2f2155367d3..ca482ea604f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java
@@ -19,7 +19,7 @@
package org.elasticsearch.cluster.metadata;
-import org.elasticsearch.cluster.routing.DjbHashFunction;
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.math.MathUtils;
@@ -43,6 +43,6 @@ public class MetaDataService extends AbstractComponent {
}
public Semaphore indexMetaDataLock(String index) {
- return indexMdLocks[MathUtils.mod(DjbHashFunction.DJB_HASH(index), indexMdLocks.length)];
+ return indexMdLocks[MathUtils.mod(Murmur3HashFunction.hash(index), indexMdLocks.length)];
}
}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/DjbHashFunction.java b/core/src/main/java/org/elasticsearch/cluster/routing/DjbHashFunction.java
deleted file mode 100644
index 7616bd382e1..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/DjbHashFunction.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing;
-
-import org.elasticsearch.cluster.routing.HashFunction;
-
-/**
- * This class implements the efficient hash function
- * developed by Daniel J. Bernstein .
- */
-public class DjbHashFunction implements HashFunction {
-
- public static int DJB_HASH(String value) {
- long hash = 5381;
-
- for (int i = 0; i < value.length(); i++) {
- hash = ((hash << 5) + hash) + value.charAt(i);
- }
-
- return (int) hash;
- }
-
- public static int DJB_HASH(byte[] value, int offset, int length) {
- long hash = 5381;
-
- final int end = offset + length;
- for (int i = offset; i < end; i++) {
- hash = ((hash << 5) + hash) + value[i];
- }
-
- return (int) hash;
- }
-
- @Override
- public int hash(String routing) {
- return DJB_HASH(routing);
- }
-
- @Override
- public int hash(String type, String id) {
- long hash = 5381;
-
- for (int i = 0; i < type.length(); i++) {
- hash = ((hash << 5) + hash) + type.charAt(i);
- }
-
- for (int i = 0; i < id.length(); i++) {
- hash = ((hash << 5) + hash) + id.charAt(i);
- }
-
- return (int) hash;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/HashFunction.java b/core/src/main/java/org/elasticsearch/cluster/routing/HashFunction.java
deleted file mode 100644
index 99977eeccb2..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/HashFunction.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing;
-
-/**
- * Simple hash function interface used for shard routing.
- */
-public interface HashFunction {
-
- /**
- * Calculate a hash value for routing
- * @param routing String to calculate the hash value from
- * @return hash value of the given routing string
- */
- int hash(String routing);
-
- /**
- * Calculate a hash value for routing and its type
- * @param type types name
- * @param id String to calculate the hash value from
- * @return hash value of the given type and routing string
- */
- @Deprecated
- int hash(String type, String id);
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java b/core/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
index 7ca602a3574..4752271ec47 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
@@ -20,15 +20,17 @@
package org.elasticsearch.cluster.routing;
import org.apache.lucene.util.StringHelper;
-import org.elasticsearch.cluster.routing.HashFunction;
/**
* Hash function based on the Murmur3 algorithm, which is the default as of Elasticsearch 2.0.
*/
-public class Murmur3HashFunction implements HashFunction {
+public final class Murmur3HashFunction {
- @Override
- public int hash(String routing) {
+ private Murmur3HashFunction() {
+ //no instance
+ }
+
+ public static int hash(String routing) {
final byte[] bytesToHash = new byte[routing.length() * 2];
for (int i = 0; i < routing.length(); ++i) {
final char c = routing.charAt(i);
@@ -37,12 +39,10 @@ public class Murmur3HashFunction implements HashFunction {
bytesToHash[i * 2] = b1;
bytesToHash[i * 2 + 1] = b2;
}
- return StringHelper.murmurhash3_x86_32(bytesToHash, 0, bytesToHash.length, 0);
+ return hash(bytesToHash, 0, bytesToHash.length);
}
- @Override
- public int hash(String type, String id) {
- throw new UnsupportedOperationException();
+ public static int hash(byte[] bytes, int offset, int length) {
+ return StringHelper.murmurhash3_x86_32(bytes, offset, length, 0);
}
-
}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 411a1ed6817..c142b754aa2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -19,7 +19,6 @@
package org.elasticsearch.cluster.routing;
-import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -47,7 +46,6 @@ import java.util.Set;
public class OperationRouting extends AbstractComponent {
-
private final AwarenessAllocationDecider awarenessAllocationDecider;
@Inject
@@ -196,9 +194,9 @@ public class OperationRouting extends AbstractComponent {
// if not, then use it as the index
String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes();
if (awarenessAttributes.length == 0) {
- return indexShard.activeInitializingShardsIt(DjbHashFunction.DJB_HASH(preference));
+ return indexShard.activeInitializingShardsIt(Murmur3HashFunction.hash(preference));
} else {
- return indexShard.preferAttributesActiveInitializingShardsIt(awarenessAttributes, nodes, DjbHashFunction.DJB_HASH(preference));
+ return indexShard.preferAttributesActiveInitializingShardsIt(awarenessAttributes, nodes, Murmur3HashFunction.hash(preference));
}
}
@@ -237,37 +235,13 @@ public class OperationRouting extends AbstractComponent {
@SuppressForbidden(reason = "Math#abs is trappy")
private int shardId(ClusterState clusterState, String index, String type, String id, @Nullable String routing) {
final IndexMetaData indexMetaData = indexMetaData(clusterState, index);
- final Version createdVersion = indexMetaData.getCreationVersion();
- final HashFunction hashFunction = indexMetaData.getRoutingHashFunction();
- final boolean useType = indexMetaData.getRoutingUseType();
-
final int hash;
if (routing == null) {
- if (!useType) {
- hash = hash(hashFunction, id);
- } else {
- hash = hash(hashFunction, type, id);
- }
+ hash = Murmur3HashFunction.hash(id);
} else {
- hash = hash(hashFunction, routing);
+ hash = Murmur3HashFunction.hash(routing);
}
- if (createdVersion.onOrAfter(Version.V_2_0_0_beta1)) {
- return MathUtils.mod(hash, indexMetaData.numberOfShards());
- } else {
- return Math.abs(hash % indexMetaData.numberOfShards());
- }
- }
-
- protected int hash(HashFunction hashFunction, String routing) {
- return hashFunction.hash(routing);
- }
-
- @Deprecated
- protected int hash(HashFunction hashFunction, String type, String id) {
- if (type == null || "_all".equals(type)) {
- throw new IllegalArgumentException("Can't route an operation with no type and having type part of the routing (for backward comp)");
- }
- return hashFunction.hash(type, id);
+ return MathUtils.mod(hash, indexMetaData.numberOfShards());
}
private void ensureNodeIdExists(DiscoveryNodes nodes, String nodeId) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index 43ad6aff1ec..596bb97887c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -19,13 +19,10 @@
package org.elasticsearch.cluster.routing;
-import com.google.common.collect.Iterators;
import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.collect.Iterators;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
/**
* A {@link RoutingNode} represents a cluster node associated with a single {@link DiscoveryNode} including all shards
@@ -51,7 +48,7 @@ public class RoutingNode implements Iterable {
@Override
public Iterator iterator() {
- return Iterators.unmodifiableIterator(shards.iterator());
+ return Collections.unmodifiableCollection(shards).iterator();
}
Iterator mutableIterator() {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index f632428e27a..d5ed922b120 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing;
import com.carrotsearch.hppc.ObjectIntHashMap;
import com.carrotsearch.hppc.cursors.ObjectCursor;
-import com.google.common.collect.Iterators;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.cluster.ClusterState;
@@ -153,7 +152,7 @@ public class RoutingNodes implements Iterable {
@Override
public Iterator iterator() {
- return Iterators.unmodifiableIterator(nodesToShards.values().iterator());
+ return Collections.unmodifiableCollection(nodesToShards.values()).iterator();
}
public RoutingTable routingTable() {
diff --git a/core/src/main/java/org/elasticsearch/common/cache/Cache.java b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
new file mode 100644
index 00000000000..d2d6970fe9e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
@@ -0,0 +1,690 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.cache;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
+
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.LongAdder;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.ToLongBiFunction;
+
+/**
+ * A simple concurrent cache.
+ *
+ * Cache is a simple concurrent cache that supports time-based and weight-based evictions, with notifications for all
+ * evictions. The design goals for this cache were simplicity and read performance. This means that we are willing to
+ * accept reduced write performance in exchange for easy-to-understand code. Cache statistics for hits, misses and
+ * evictions are exposed.
+ *
+ * The design of the cache is relatively simple. The cache is segmented into 256 segments which are backed by HashMaps.
+ * Each segment is protected by a re-entrant read/write lock. The read/write locks permit multiple concurrent readers
+ * without contention, and the segments gives us write throughput without impacting readers (so readers are blocked only
+ * if they are reading a segment that a writer is writing to).
+ *
+ * The LRU functionality is backed by a single doubly-linked list chaining the entries in order of insertion. This
+ * LRU list is protected by a lock that serializes all writes to it. There are opportunities for improvements
+ * here if write throughput is a concern.
+ *
+ * LRU list mutations could be inserted into a blocking queue that a single thread is reading from
+ * and applying to the LRU list.
+ * Promotions could be deferred for entries that were "recently" promoted.
+ * Locks on the list could be taken per node being modified instead of globally.
+ *
+ *
+ * Evictions only occur after a mutation to the cache (meaning an entry promotion, a cache insertion, or a manual
+ * invalidation) or an explicit call to {@link #refresh()}.
+ *
+ * @param The type of the keys
+ * @param The type of the values
+ */
+public class Cache {
+ // positive if entries have an expiration
+ private long expireAfterAccess = -1;
+
+ // true if entries can expire after access
+ private boolean entriesExpireAfterAccess;
+
+ // positive if entries have an expiration after write
+ private long expireAfterWrite = -1;
+
+ // true if entries can expire after initial insertion
+ private boolean entriesExpireAfterWrite;
+
+ // the number of entries in the cache
+ private int count = 0;
+
+ // the weight of the entries in the cache
+ private long weight = 0;
+
+ // the maximum weight that this cache supports
+ private long maximumWeight = -1;
+
+ // the weigher of entries
+ private ToLongBiFunction weigher = (k, v) -> 1;
+
+ // the removal callback
+ private RemovalListener removalListener = notification -> {
+ };
+
+ // use CacheBuilder to construct
+ Cache() {
+ }
+
+ void setExpireAfterAccess(long expireAfterAccess) {
+ if (expireAfterAccess <= 0) {
+ throw new IllegalArgumentException("expireAfterAccess <= 0");
+ }
+ this.expireAfterAccess = expireAfterAccess;
+ this.entriesExpireAfterAccess = true;
+ }
+
+ void setExpireAfterWrite(long expireAfterWrite) {
+ if (expireAfterWrite <= 0) {
+ throw new IllegalArgumentException("expireAfterWrite <= 0");
+ }
+ this.expireAfterWrite = expireAfterWrite;
+ this.entriesExpireAfterWrite = true;
+ }
+
+ void setMaximumWeight(long maximumWeight) {
+ if (maximumWeight < 0) {
+ throw new IllegalArgumentException("maximumWeight < 0");
+ }
+ this.maximumWeight = maximumWeight;
+ }
+
+ void setWeigher(ToLongBiFunction weigher) {
+ Objects.requireNonNull(weigher);
+ this.weigher = weigher;
+ }
+
+ void setRemovalListener(RemovalListener removalListener) {
+ Objects.requireNonNull(removalListener);
+ this.removalListener = removalListener;
+ }
+
+ /**
+ * The relative time used to track time-based evictions.
+ *
+ * @return the current relative time
+ */
+ protected long now() {
+ // System.nanoTime takes non-negligible time, so we only use it if we need it
+ // use System.nanoTime because we want relative time, not absolute time
+ return entriesExpireAfterAccess || entriesExpireAfterWrite ? System.nanoTime() : 0;
+ }
+
+ // the state of an entry in the LRU list
+ enum State {
+ NEW, EXISTING, DELETED
+ }
+
+ static class Entry {
+ final K key;
+ final V value;
+ long writeTime;
+ volatile long accessTime;
+ Entry before;
+ Entry after;
+ State state = State.NEW;
+
+ public Entry(K key, V value, long writeTime) {
+ this.key = key;
+ this.value = value;
+ this.writeTime = this.accessTime = writeTime;
+ }
+ }
+
+ /**
+ * A cache segment.
+ *
+ * A CacheSegment is backed by a HashMap and is protected by a read/write lock.
+ *
+ * @param the type of the keys
+ * @param the type of the values
+ */
+ private static class CacheSegment {
+ // read/write lock protecting mutations to the segment
+ ReadWriteLock segmentLock = new ReentrantReadWriteLock();
+
+ ReleasableLock readLock = new ReleasableLock(segmentLock.readLock());
+ ReleasableLock writeLock = new ReleasableLock(segmentLock.writeLock());
+
+ Map> map = new HashMap<>();
+ SegmentStats segmentStats = new SegmentStats();
+
+ /**
+ * get an entry from the segment
+ *
+ * @param key the key of the entry to get from the cache
+ * @param now the access time of this entry
+ * @return the entry if there was one, otherwise null
+ */
+ Entry get(K key, long now) {
+ Entry entry;
+ try (ReleasableLock ignored = readLock.acquire()) {
+ entry = map.get(key);
+ }
+ if (entry != null) {
+ segmentStats.hit();
+ entry.accessTime = now;
+ } else {
+ segmentStats.miss();
+ }
+ return entry;
+ }
+
+ /**
+ * put an entry into the segment
+ *
+ * @param key the key of the entry to add to the cache
+ * @param value the value of the entry to add to the cache
+ * @param now the access time of this entry
+ * @return a tuple of the new entry and the existing entry, if there was one otherwise null
+ */
+ Tuple, Entry> put(K key, V value, long now) {
+ Entry entry = new Entry<>(key, value, now);
+ Entry existing;
+ try (ReleasableLock ignored = writeLock.acquire()) {
+ existing = map.put(key, entry);
+ }
+ return Tuple.tuple(entry, existing);
+ }
+
+ /**
+ * remove an entry from the segment
+ *
+ * @param key the key of the entry to remove from the cache
+ * @return the removed entry if there was one, otherwise null
+ */
+ Entry remove(K key) {
+ Entry entry;
+ try (ReleasableLock ignored = writeLock.acquire()) {
+ entry = map.remove(key);
+ }
+ if (entry != null) {
+ segmentStats.eviction();
+ }
+ return entry;
+ }
+
+ private static class SegmentStats {
+ private final LongAdder hits = new LongAdder();
+ private final LongAdder misses = new LongAdder();
+ private final LongAdder evictions = new LongAdder();
+
+ void hit() {
+ hits.increment();
+ }
+
+ void miss() {
+ misses.increment();
+ }
+
+ void eviction() {
+ evictions.increment();
+ }
+ }
+ }
+
+ public static final int NUMBER_OF_SEGMENTS = 256;
+ private final CacheSegment[] segments = new CacheSegment[NUMBER_OF_SEGMENTS];
+
+ {
+ for (int i = 0; i < segments.length; i++) {
+ segments[i] = new CacheSegment<>();
+ }
+ }
+
+ Entry head;
+ Entry tail;
+
+ // lock protecting mutations to the LRU list
+ private ReleasableLock lruLock = new ReleasableLock(new ReentrantLock());
+
+ /**
+ * Returns the value to which the specified key is mapped, or null if this map contains no mapping for the key.
+ *
+ * @param key the key whose associated value is to be returned
+ * @return the value to which the specified key is mapped, or null if this map contains no mapping for the key
+ */
+ public V get(K key) {
+ return get(key, now());
+ }
+
+ private V get(K key, long now) {
+ CacheSegment segment = getCacheSegment(key);
+ Entry entry = segment.get(key, now);
+ if (entry == null || isExpired(entry, now)) {
+ return null;
+ } else {
+ promote(entry, now);
+ return entry.value;
+ }
+ }
+
+ /**
+ * If the specified key is not already associated with a value (or is mapped to null), attempts to compute its
+ * value using the given mapping function and enters it into this map unless null.
+ *
+ * @param key the key whose associated value is to be returned or computed for if non-existant
+ * @param loader the function to compute a value given a key
+ * @return the current (existing or computed) value associated with the specified key, or null if the computed
+ * value is null
+ * @throws ExecutionException thrown if loader throws an exception
+ */
+ public V computeIfAbsent(K key, CacheLoader loader) throws ExecutionException {
+ long now = now();
+ V value = get(key, now);
+ if (value == null) {
+ CacheSegment segment = getCacheSegment(key);
+ // we synchronize against the segment lock; this is to avoid a scenario where another thread is inserting
+ // a value for the same key via put which would not be observed on this thread without a mechanism
+ // synchronizing the two threads; it is possible that the segment lock will be too expensive here (it blocks
+ // readers too!) so consider this as a possible place to optimize should contention be observed
+ try (ReleasableLock ignored = segment.writeLock.acquire()) {
+ value = get(key, now);
+ if (value == null) {
+ try {
+ value = loader.load(key);
+ } catch (Exception e) {
+ throw new ExecutionException(e);
+ }
+ if (value == null) {
+ throw new ExecutionException(new NullPointerException("loader returned a null value"));
+ }
+ put(key, value, now);
+ }
+ }
+ }
+ return value;
+ }
+
+ /**
+ * Associates the specified value with the specified key in this map. If the map previously contained a mapping for
+ * the key, the old value is replaced.
+ *
+ * @param key key with which the specified value is to be associated
+ * @param value value to be associated with the specified key
+ */
+ public void put(K key, V value) {
+ long now = now();
+ put(key, value, now);
+ }
+
+ private void put(K key, V value, long now) {
+ CacheSegment segment = getCacheSegment(key);
+ Tuple, Entry> tuple = segment.put(key, value, now);
+ boolean replaced = false;
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ if (tuple.v2() != null && tuple.v2().state == State.EXISTING) {
+ if (unlink(tuple.v2())) {
+ replaced = true;
+ }
+ }
+ promote(tuple.v1(), now);
+ }
+ if (replaced) {
+ removalListener.onRemoval(new RemovalNotification(tuple.v2().key, tuple.v2().value, RemovalNotification.RemovalReason.REPLACED));
+ }
+ }
+
+ /**
+ * Invalidate the association for the specified key. A removal notification will be issued for invalidated
+ * entries with {@link org.elasticsearch.common.cache.RemovalNotification.RemovalReason} INVALIDATED.
+ *
+ * @param key the key whose mapping is to be invalidated from the cache
+ */
+ public void invalidate(K key) {
+ CacheSegment segment = getCacheSegment(key);
+ Entry entry = segment.remove(key);
+ if (entry != null) {
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ delete(entry, RemovalNotification.RemovalReason.INVALIDATED);
+ }
+ }
+ }
+
+ /**
+ * Invalidate all cache entries. A removal notification will be issued for invalidated entries with
+ * {@link org.elasticsearch.common.cache.RemovalNotification.RemovalReason} INVALIDATED.
+ */
+ public void invalidateAll() {
+ Entry h;
+
+ boolean[] haveSegmentLock = new boolean[NUMBER_OF_SEGMENTS];
+ try {
+ for (int i = 0; i < NUMBER_OF_SEGMENTS; i++) {
+ segments[i].segmentLock.writeLock().lock();
+ haveSegmentLock[i] = true;
+ }
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ h = head;
+ Arrays.stream(segments).forEach(segment -> segment.map = new HashMap<>());
+ Entry current = head;
+ while (current != null) {
+ current.state = State.DELETED;
+ current = current.after;
+ }
+ head = tail = null;
+ count = 0;
+ weight = 0;
+ }
+ } finally {
+ for (int i = NUMBER_OF_SEGMENTS - 1; i >= 0; i--) {
+ if (haveSegmentLock[i]) {
+ segments[i].segmentLock.writeLock().unlock();
+ }
+ }
+ }
+ while (h != null) {
+ removalListener.onRemoval(new RemovalNotification<>(h.key, h.value, RemovalNotification.RemovalReason.INVALIDATED));
+ h = h.after;
+ }
+ }
+
+ /**
+ * Force any outstanding size-based and time-based evictions to occur
+ */
+ public void refresh() {
+ long now = now();
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ evict(now);
+ }
+ }
+
+ /**
+ * The number of entries in the cache.
+ *
+ * @return the number of entries in the cache
+ */
+ public int count() {
+ return count;
+ }
+
+ /**
+ * The weight of the entries in the cache.
+ *
+ * @return the weight of the entries in the cache
+ */
+ public long weight() {
+ return weight;
+ }
+
+ /**
+ * An LRU sequencing of the keys in the cache that supports removal. This sequence is not protected from mutations
+ * to the cache (except for {@link Iterator#remove()}. The result of iteration under any other mutation is
+ * undefined.
+ *
+ * @return an LRU-ordered {@link Iterable} over the keys in the cache
+ */
+ public Iterable keys() {
+ return () -> new Iterator() {
+ private CacheIterator iterator = new CacheIterator(head);
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public K next() {
+ return iterator.next().key;
+ }
+
+ @Override
+ public void remove() {
+ iterator.remove();
+ }
+ };
+ }
+
+ /**
+ * An LRU sequencing of the values in the cache. This sequence is not protected from mutations
+ * to the cache. The result of iteration under mutation is undefined.
+ *
+ * @return an LRU-ordered {@link Iterable} over the values in the cache
+ */
+ public Iterable values() {
+ return () -> new Iterator() {
+ private CacheIterator iterator = new CacheIterator(head);
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public V next() {
+ return iterator.next().value;
+ }
+ };
+ }
+
+ private class CacheIterator implements Iterator> {
+ private Entry current;
+ private Entry next;
+
+ CacheIterator(Entry head) {
+ current = null;
+ next = head;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return next != null;
+ }
+
+ @Override
+ public Entry next() {
+ current = next;
+ next = next.after;
+ return current;
+ }
+
+ @Override
+ public void remove() {
+ Entry entry = current;
+ if (entry != null) {
+ CacheSegment segment = getCacheSegment(entry.key);
+ segment.remove(entry.key);
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ current = null;
+ delete(entry, RemovalNotification.RemovalReason.INVALIDATED);
+ }
+ }
+ }
+ }
+
+ /**
+ * The cache statistics tracking hits, misses and evictions. These are taken on a best-effort basis meaning that
+ * they could be out-of-date mid-flight.
+ *
+ * @return the current cache statistics
+ */
+ public CacheStats stats() {
+ long hits = 0;
+ long misses = 0;
+ long evictions = 0;
+ for (int i = 0; i < segments.length; i++) {
+ hits += segments[i].segmentStats.hits.longValue();
+ misses += segments[i].segmentStats.misses.longValue();
+ evictions += segments[i].segmentStats.evictions.longValue();
+ }
+ return new CacheStats(hits, misses, evictions);
+ }
+
+ public static class CacheStats {
+ private long hits;
+ private long misses;
+ private long evictions;
+
+ public CacheStats(long hits, long misses, long evictions) {
+ this.hits = hits;
+ this.misses = misses;
+ this.evictions = evictions;
+ }
+
+ public long getHits() {
+ return hits;
+ }
+
+ public long getMisses() {
+ return misses;
+ }
+
+ public long getEvictions() {
+ return evictions;
+ }
+ }
+
+ private boolean promote(Entry entry, long now) {
+ boolean promoted = true;
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ switch (entry.state) {
+ case DELETED:
+ promoted = false;
+ break;
+ case EXISTING:
+ relinkAtHead(entry);
+ break;
+ case NEW:
+ linkAtHead(entry);
+ break;
+ }
+ if (promoted) {
+ evict(now);
+ }
+ }
+ return promoted;
+ }
+
+ private void evict(long now) {
+ assert lruLock.isHeldByCurrentThread();
+
+ while (tail != null && shouldPrune(tail, now)) {
+ CacheSegment segment = getCacheSegment(tail.key);
+ Entry entry = tail;
+ if (segment != null) {
+ segment.remove(tail.key);
+ }
+ delete(entry, RemovalNotification.RemovalReason.EVICTED);
+ }
+ }
+
+ private void delete(Entry entry, RemovalNotification.RemovalReason removalReason) {
+ assert lruLock.isHeldByCurrentThread();
+
+ if (unlink(entry)) {
+ removalListener.onRemoval(new RemovalNotification<>(entry.key, entry.value, removalReason));
+ }
+ }
+
+ private boolean shouldPrune(Entry entry, long now) {
+ return exceedsWeight() || isExpired(entry, now);
+ }
+
+ private boolean exceedsWeight() {
+ return maximumWeight != -1 && weight > maximumWeight;
+ }
+
+ private boolean isExpired(Entry entry, long now) {
+ return (entriesExpireAfterAccess && now - entry.accessTime > expireAfterAccess) ||
+ (entriesExpireAfterWrite && now - entry.writeTime > expireAfterWrite);
+ }
+
+ private boolean unlink(Entry entry) {
+ assert lruLock.isHeldByCurrentThread();
+
+ if (entry.state == State.EXISTING) {
+ final Entry before = entry.before;
+ final Entry after = entry.after;
+
+ if (before == null) {
+ // removing the head
+ assert head == entry;
+ head = after;
+ if (head != null) {
+ head.before = null;
+ }
+ } else {
+ // removing inner element
+ before.after = after;
+ entry.before = null;
+ }
+
+ if (after == null) {
+ // removing tail
+ assert tail == entry;
+ tail = before;
+ if (tail != null) {
+ tail.after = null;
+ }
+ } else {
+ // removing inner element
+ after.before = before;
+ entry.after = null;
+ }
+
+ count--;
+ weight -= weigher.applyAsLong(entry.key, entry.value);
+ entry.state = State.DELETED;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void linkAtHead(Entry entry) {
+ assert lruLock.isHeldByCurrentThread();
+
+ Entry h = head;
+ entry.before = null;
+ entry.after = head;
+ head = entry;
+ if (h == null) {
+ tail = entry;
+ } else {
+ h.before = entry;
+ }
+
+ count++;
+ weight += weigher.applyAsLong(entry.key, entry.value);
+ entry.state = State.EXISTING;
+ }
+
+ private void relinkAtHead(Entry entry) {
+ assert lruLock.isHeldByCurrentThread();
+
+ if (head != entry) {
+ unlink(entry);
+ linkAtHead(entry);
+ }
+ }
+
+ private CacheSegment getCacheSegment(K key) {
+ return segments[key.hashCode() & 0xff];
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java b/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java
new file mode 100644
index 00000000000..ffb0e591180
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/cache/CacheBuilder.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.cache;
+
+import java.util.Objects;
+import java.util.function.ToLongBiFunction;
+
+public class CacheBuilder {
+ private long maximumWeight = -1;
+ private long expireAfterAccess = -1;
+ private long expireAfterWrite = -1;
+ private ToLongBiFunction weigher;
+ private RemovalListener removalListener;
+
+ public static CacheBuilder builder() {
+ return new CacheBuilder<>();
+ }
+
+ private CacheBuilder() {
+ }
+
+ public CacheBuilder setMaximumWeight(long maximumWeight) {
+ if (maximumWeight < 0) {
+ throw new IllegalArgumentException("maximumWeight < 0");
+ }
+ this.maximumWeight = maximumWeight;
+ return this;
+ }
+
+ public CacheBuilder setExpireAfterAccess(long expireAfterAccess) {
+ if (expireAfterAccess <= 0) {
+ throw new IllegalArgumentException("expireAfterAccess <= 0");
+ }
+ this.expireAfterAccess = expireAfterAccess;
+ return this;
+ }
+
+ public CacheBuilder setExpireAfterWrite(long expireAfterWrite) {
+ if (expireAfterWrite <= 0) {
+ throw new IllegalArgumentException("expireAfterWrite <= 0");
+ }
+ this.expireAfterWrite = expireAfterWrite;
+ return this;
+ }
+
+ public CacheBuilder weigher(ToLongBiFunction weigher) {
+ Objects.requireNonNull(weigher);
+ this.weigher = weigher;
+ return this;
+ }
+
+ public CacheBuilder removalListener(RemovalListener removalListener) {
+ Objects.requireNonNull(removalListener);
+ this.removalListener = removalListener;
+ return this;
+ }
+
+ public Cache build() {
+ Cache cache = new Cache();
+ if (maximumWeight != -1) {
+ cache.setMaximumWeight(maximumWeight);
+ }
+ if (expireAfterAccess != -1) {
+ cache.setExpireAfterAccess(expireAfterAccess);
+ }
+ if (expireAfterWrite != -1) {
+ cache.setExpireAfterWrite(expireAfterWrite);
+ }
+ if (weigher != null) {
+ cache.setWeigher(weigher);
+ }
+ if (removalListener != null) {
+ cache.setRemovalListener(removalListener);
+ }
+ return cache;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/SimpleHashFunction.java b/core/src/main/java/org/elasticsearch/common/cache/CacheLoader.java
similarity index 65%
rename from core/src/main/java/org/elasticsearch/cluster/routing/SimpleHashFunction.java
rename to core/src/main/java/org/elasticsearch/common/cache/CacheLoader.java
index bbb6a6174da..85636e1e186 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/SimpleHashFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/CacheLoader.java
@@ -17,20 +17,9 @@
* under the License.
*/
-package org.elasticsearch.cluster.routing;
+package org.elasticsearch.common.cache;
-/**
- * This class implements a simple hash function based on Java Build-In {@link Object#hashCode()}
- */
-public class SimpleHashFunction implements HashFunction {
-
- @Override
- public int hash(String routing) {
- return routing.hashCode();
- }
-
- @Override
- public int hash(String type, String id) {
- return type.hashCode() + 31 * id.hashCode();
- }
+@FunctionalInterface
+public interface CacheLoader {
+ V load(K key) throws Exception;
}
diff --git a/core/src/main/java/org/elasticsearch/index/LocalNodeIdModule.java b/core/src/main/java/org/elasticsearch/common/cache/RemovalListener.java
similarity index 64%
rename from core/src/main/java/org/elasticsearch/index/LocalNodeIdModule.java
rename to core/src/main/java/org/elasticsearch/common/cache/RemovalListener.java
index 82e36cd6efb..ae133000f76 100644
--- a/core/src/main/java/org/elasticsearch/index/LocalNodeIdModule.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/RemovalListener.java
@@ -17,23 +17,9 @@
* under the License.
*/
-package org.elasticsearch.index;
+package org.elasticsearch.common.cache;
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- *
- */
-public class LocalNodeIdModule extends AbstractModule {
-
- private final String localNodeId;
-
- public LocalNodeIdModule(String localNodeId) {
- this.localNodeId = localNodeId;
- }
-
- @Override
- protected void configure() {
- bind(String.class).annotatedWith(LocalNodeId.class).toInstance(localNodeId);
- }
-}
+@FunctionalInterface
+public interface RemovalListener {
+ void onRemoval(RemovalNotification notification);
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/LocalNodeId.java b/core/src/main/java/org/elasticsearch/common/cache/RemovalNotification.java
similarity index 56%
rename from core/src/main/java/org/elasticsearch/index/LocalNodeId.java
rename to core/src/main/java/org/elasticsearch/common/cache/RemovalNotification.java
index a045636a688..afea5a54480 100644
--- a/core/src/main/java/org/elasticsearch/index/LocalNodeId.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/RemovalNotification.java
@@ -17,24 +17,30 @@
* under the License.
*/
-package org.elasticsearch.index;
+package org.elasticsearch.common.cache;
-import org.elasticsearch.common.inject.BindingAnnotation;
+public class RemovalNotification {
+ public enum RemovalReason {REPLACED, INVALIDATED, EVICTED}
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
+ private final K key;
+ private final V value;
+ private final RemovalReason removalReason;
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
+ public RemovalNotification(K key, V value, RemovalReason removalReason) {
+ this.key = key;
+ this.value = value;
+ this.removalReason = removalReason;
+ }
-/**
- *
- */
-@BindingAnnotation
-@Target({FIELD, PARAMETER})
-@Retention(RUNTIME)
-@Documented
-public @interface LocalNodeId {
+ public K getKey() {
+ return key;
+ }
+
+ public V getValue() {
+ return value;
+ }
+
+ public RemovalReason getRemovalReason() {
+ return removalReason;
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/common/collect/EvictingQueue.java b/core/src/main/java/org/elasticsearch/common/collect/EvictingQueue.java
new file mode 100644
index 00000000000..51cc08d0209
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/collect/EvictingQueue.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.collect;
+
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Queue;
+
+/**
+ * An {@code EvictingQueue} is a non-blocking queue which is limited to a maximum size; when new elements are added to a
+ * full queue, elements are evicted from the head of the queue to accommodate the new elements.
+ *
+ * @param The type of elements in the queue.
+ */
+public class EvictingQueue implements Queue {
+ private final int maximumSize;
+ private final ArrayDeque queue;
+
+ /**
+ * Construct a new {@code EvictingQueue} that holds {@code maximumSize} elements.
+ *
+ * @param maximumSize The maximum number of elements that the queue can hold
+ * @throws IllegalArgumentException if {@code maximumSize} is less than zero
+ */
+ public EvictingQueue(int maximumSize) {
+ if (maximumSize < 0) {
+ throw new IllegalArgumentException("maximumSize < 0");
+ }
+ this.maximumSize = maximumSize;
+ this.queue = new ArrayDeque<>(maximumSize);
+ }
+
+ /**
+ * @return the number of additional elements that the queue can accommodate before evictions occur
+ */
+ public int remainingCapacity() {
+ return this.maximumSize - this.size();
+ }
+
+ /**
+ * Add the given element to the queue, possibly forcing an eviction from the head if {@link #remainingCapacity()} is
+ * zero.
+ *
+ * @param t the element to add
+ * @return true if the element was added (always the case for {@code EvictingQueue}
+ */
+ @Override
+ public boolean add(T t) {
+ if (maximumSize == 0) {
+ return true;
+ }
+ if (queue.size() == maximumSize) {
+ queue.remove();
+ }
+ queue.add(t);
+ return true;
+ }
+
+ /**
+ * @see #add(Object)
+ */
+ @Override
+ public boolean offer(T t) {
+ return add(t);
+ }
+
+ @Override
+ public T remove() {
+ return queue.remove();
+ }
+
+
+ @Override
+ public T poll() {
+ return queue.poll();
+ }
+
+ @Override
+ public T element() {
+ return queue.element();
+ }
+
+ @Override
+ public T peek() {
+ return queue.peek();
+ }
+
+ @Override
+ public int size() {
+ return queue.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return queue.isEmpty();
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ return queue.contains(o);
+ }
+
+ @Override
+ public Iterator iterator() {
+ return queue.iterator();
+ }
+
+ @Override
+ public Object[] toArray() {
+ return queue.toArray();
+ }
+
+ @Override
+ public T1[] toArray(T1[] a) {
+ return queue.toArray(a);
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ return queue.remove(o);
+ }
+
+ @Override
+ public boolean containsAll(Collection c) {
+ return queue.containsAll(c);
+ }
+
+ /**
+ * Add the given elements to the queue, possibly forcing evictions from the head if {@link #remainingCapacity()} is
+ * zero or becomes zero during the execution of this method.
+ *
+ * @param c the collection of elements to add
+ * @return true if any elements were added to the queue
+ */
+ @Override
+ public boolean addAll(Collection c) {
+ boolean modified = false;
+ for (T e : c)
+ if (add(e))
+ modified = true;
+ return modified;
+ }
+
+ @Override
+ public boolean removeAll(Collection c) {
+ return queue.removeAll(c);
+ }
+
+ @Override
+ public boolean retainAll(Collection c) {
+ return queue.retainAll(c);
+ }
+
+ @Override
+ public void clear() {
+ queue.clear();
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
new file mode 100644
index 00000000000..34546120b0a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.collect;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+public class Iterators {
+ public static Iterator concat(Iterator... iterators) {
+ if (iterators == null) {
+ throw new NullPointerException("iterators");
+ }
+
+ return new ConcatenatedIterator<>(iterators);
+ }
+
+ static class ConcatenatedIterator implements Iterator {
+ private final Iterator[] iterators;
+ private int index = 0;
+
+ public ConcatenatedIterator(Iterator... iterators) {
+ if (iterators == null) {
+ throw new NullPointerException("iterators");
+ }
+ for (int i = 0; i < iterators.length; i++) {
+ if (iterators[i] == null) {
+ throw new NullPointerException("iterators[" + i + "]");
+ }
+ }
+ this.iterators = iterators;
+ }
+
+ @Override
+ public boolean hasNext() {
+ boolean hasNext = false;
+ while (index < iterators.length && !(hasNext = iterators[index].hasNext())) {
+ index++;
+ }
+
+ return hasNext;
+ }
+
+ @Override
+ public T next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return iterators[index].next();
+ }
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index c50b85a835c..0bc9455bea3 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -19,12 +19,6 @@
package org.elasticsearch.common.geo;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
-
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.XGeoHashUtils;
import org.apache.lucene.util.XGeoUtils;
@@ -32,15 +26,12 @@ import org.apache.lucene.util.XGeoUtils;
/**
*
*/
-public final class GeoPoint implements Writeable {
+public final class GeoPoint {
private double lat;
private double lon;
private final static double TOLERANCE = XGeoUtils.TOLERANCE;
- // for serialization purposes
- private static final GeoPoint PROTOTYPE = new GeoPoint(Double.NaN, Double.NaN);
-
public GeoPoint() {
}
@@ -179,21 +170,4 @@ public final class GeoPoint implements Writeable {
public static GeoPoint fromIndexLong(long indexLong) {
return new GeoPoint().resetFromIndexHash(indexLong);
}
-
- @Override
- public GeoPoint readFrom(StreamInput in) throws IOException {
- double lat = in.readDouble();
- double lon = in.readDouble();
- return new GeoPoint(lat, lon);
- }
-
- public static GeoPoint readGeoPointFrom(StreamInput in) throws IOException {
- return PROTOTYPE.readFrom(in);
- }
-
- @Override
- public void writeTo(StreamOutput out) throws IOException {
- out.writeDouble(lat);
- out.writeDouble(lon);
- }
}
diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java b/core/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java
index aad0c3a5ef3..bb057d6040d 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright (C) 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.elasticsearch.common.inject.internal;
import java.lang.annotation.Annotation;
diff --git a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
index cb5ca5fec64..e53e7a73eb7 100644
--- a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
@@ -19,9 +19,8 @@
package org.elasticsearch.common.io;
-import com.google.common.collect.Iterators;
-
import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.logging.ESLogger;
import java.io.BufferedReader;
@@ -35,6 +34,7 @@ import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.StreamSupport;
import static java.nio.file.FileVisitResult.CONTINUE;
import static java.nio.file.FileVisitResult.SKIP_SUBTREE;
@@ -328,7 +328,7 @@ public final class FileSystemUtils {
*/
public static Path[] files(Path from, DirectoryStream.Filter filter) throws IOException {
try (DirectoryStream stream = Files.newDirectoryStream(from, filter)) {
- return Iterators.toArray(stream.iterator(), Path.class);
+ return toArray(stream);
}
}
@@ -337,7 +337,7 @@ public final class FileSystemUtils {
*/
public static Path[] files(Path directory) throws IOException {
try (DirectoryStream stream = Files.newDirectoryStream(directory)) {
- return Iterators.toArray(stream.iterator(), Path.class);
+ return toArray(stream);
}
}
@@ -346,8 +346,12 @@ public final class FileSystemUtils {
*/
public static Path[] files(Path directory, String glob) throws IOException {
try (DirectoryStream stream = Files.newDirectoryStream(directory, glob)) {
- return Iterators.toArray(stream.iterator(), Path.class);
+ return toArray(stream);
}
}
+ private static Path[] toArray(DirectoryStream stream) {
+ return StreamSupport.stream(stream.spliterator(), false).toArray(length -> new Path[length]);
+ }
+
}
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 17d99951209..a6fc0914dbe 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.text.StringAndBytesText;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilder;
@@ -336,19 +337,6 @@ public abstract class StreamInput extends InputStream {
@Override
public abstract void close() throws IOException;
-// // IS
-//
-// @Override public int read() throws IOException {
-// return readByte();
-// }
-//
-// // Here, we assume that we always can read the full byte array
-//
-// @Override public int read(byte[] b, int off, int len) throws IOException {
-// readBytes(b, off, len);
-// return len;
-// }
-
public String[] readStringArray() throws IOException {
int size = readVInt();
if (size == 0) {
@@ -449,11 +437,20 @@ public abstract class StreamInput extends InputStream {
return readDoubleArray();
case 21:
return readBytesRef();
+ case 22:
+ return readGeoPoint();
default:
throw new IOException("Can't read unknown type [" + type + "]");
}
}
+ /**
+ * Reads a {@link GeoPoint} from this stream input
+ */
+ public GeoPoint readGeoPoint() throws IOException {
+ return new GeoPoint(readDouble(), readDouble());
+ }
+
public int[] readIntArray() throws IOException {
int length = readVInt();
int[] values = new int[length];
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 16128e40c64..3e4aabb6284 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -30,6 +30,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
@@ -422,6 +423,9 @@ public abstract class StreamOutput extends OutputStream {
} else if (value instanceof BytesRef) {
writeByte((byte) 21);
writeBytesRef((BytesRef) value);
+ } else if (type == GeoPoint.class) {
+ writeByte((byte) 22);
+ writeGeoPoint((GeoPoint) value);
} else {
throw new IOException("Can't write type [" + type + "]");
}
@@ -467,14 +471,6 @@ public abstract class StreamOutput extends OutputStream {
}
}
- private static int parseIntSafe(String val, int defaultVal) {
- try {
- return Integer.parseInt(val);
- } catch (NumberFormatException ex) {
- return defaultVal;
- }
- }
-
public void writeThrowable(Throwable throwable) throws IOException {
if (throwable == null) {
writeBoolean(false);
@@ -596,4 +592,12 @@ public abstract class StreamOutput extends OutputStream {
public void writeScoreFunction(ScoreFunctionBuilder scoreFunctionBuilder) throws IOException {
writeNamedWriteable(scoreFunctionBuilder);
}
+
+ /**
+ * Writes the given {@link GeoPoint} to the stream
+ */
+ public void writeGeoPoint(GeoPoint geoPoint) throws IOException {
+ writeDouble(geoPoint.lat());
+ writeDouble(geoPoint.lon());
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 060482e2e8a..3aaaf9677b8 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -46,14 +46,11 @@ import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.text.ParseException;
import java.util.*;
-import static org.elasticsearch.common.lucene.search.NoopCollector.NOOP_COLLECTOR;
-
/**
*
*/
@@ -229,27 +226,6 @@ public class Lucene {
}.run();
}
- public static long count(IndexSearcher searcher, Query query) throws IOException {
- return searcher.count(query);
- }
-
- /**
- * Performs a count on the searcher for query. Terminates
- * early when the count has reached terminateAfter
- */
- public static long count(IndexSearcher searcher, Query query, int terminateAfterCount) throws IOException {
- EarlyTerminatingCollector countCollector = createCountBasedEarlyTerminatingCollector(terminateAfterCount);
- countWithEarlyTermination(searcher, query, countCollector);
- return countCollector.count();
- }
-
- /**
- * Creates count based early termination collector with a threshold of maxCountHits
- */
- public final static EarlyTerminatingCollector createCountBasedEarlyTerminatingCollector(int maxCountHits) {
- return new EarlyTerminatingCollector(maxCountHits);
- }
-
/**
* Wraps delegate with count based early termination collector with a threshold of maxCountHits
*/
@@ -265,99 +241,27 @@ public class Lucene {
}
/**
- * Performs an exists (count > 0) query on the searcher for query
- * with filter using the given collector
- *
- * The collector can be instantiated using Lucene.createExistsCollector()
+ * Check whether there is one or more documents matching the provided query.
*/
- public static boolean exists(IndexSearcher searcher, Query query, Filter filter,
- EarlyTerminatingCollector collector) throws IOException {
- collector.reset();
- countWithEarlyTermination(searcher, filter, query, collector);
- return collector.exists();
- }
-
-
- /**
- * Performs an exists (count > 0) query on the searcher for query
- * using the given collector
- *
- * The collector can be instantiated using Lucene.createExistsCollector()
- */
- public static boolean exists(IndexSearcher searcher, Query query, EarlyTerminatingCollector collector) throws IOException {
- collector.reset();
- countWithEarlyTermination(searcher, query, collector);
- return collector.exists();
- }
-
- /**
- * Calls countWithEarlyTermination(searcher, null, query, collector)
- */
- public static boolean countWithEarlyTermination(IndexSearcher searcher, Query query,
- EarlyTerminatingCollector collector) throws IOException {
- return countWithEarlyTermination(searcher, null, query, collector);
- }
-
- /**
- * Performs a count on query and filter with early termination using searcher.
- * The early termination threshold is specified by the provided collector
- */
- public static boolean countWithEarlyTermination(IndexSearcher searcher, Filter filter, Query query,
- EarlyTerminatingCollector collector) throws IOException {
- try {
- if (filter == null) {
- searcher.search(query, collector);
- } else {
- searcher.search(query, filter, collector);
+ public static boolean exists(IndexSearcher searcher, Query query) throws IOException {
+ final Weight weight = searcher.createNormalizedWeight(query, false);
+ // the scorer API should be more efficient at stopping after the first
+ // match than the bulk scorer API
+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
+ final Scorer scorer = weight.scorer(context);
+ if (scorer == null) {
+ continue;
+ }
+ final Bits liveDocs = context.reader().getLiveDocs();
+ for (int doc = scorer.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = scorer.nextDoc()) {
+ if (liveDocs == null || liveDocs.get(doc)) {
+ return true;
+ }
}
- } catch (EarlyTerminationException e) {
- // early termination
- return true;
}
return false;
}
- /**
- * Performs an exists (count > 0) query on the searcher from the searchContext for query
- * using the given collector
- *
- * The collector can be instantiated using Lucene.createExistsCollector()
- */
- public static boolean exists(SearchContext searchContext, Query query, EarlyTerminatingCollector collector) throws IOException {
- collector.reset();
- try {
- searchContext.searcher().search(query, collector);
- } catch (EarlyTerminationException e) {
- // ignore, just early termination...
- } finally {
- searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
- }
- return collector.exists();
- }
-
- /**
- * Creates an {@link org.elasticsearch.common.lucene.Lucene.EarlyTerminatingCollector}
- * with a threshold of 1
- */
- public final static EarlyTerminatingCollector createExistsCollector() {
- return createCountBasedEarlyTerminatingCollector(1);
- }
-
- /**
- * Closes the index writer, returning false if it failed to close.
- */
- public static boolean safeClose(IndexWriter writer) {
- if (writer == null) {
- return true;
- }
- try {
- writer.close();
- return true;
- } catch (Throwable e) {
- return false;
- }
- }
-
public static TopDocs readTopDocs(StreamInput in) throws IOException {
if (in.readBoolean()) {
int totalHits = in.readVInt();
@@ -612,19 +516,11 @@ public class Lucene {
private int count = 0;
private LeafCollector leafCollector;
- EarlyTerminatingCollector(int maxCountHits) {
- this.maxCountHits = maxCountHits;
- this.delegate = NOOP_COLLECTOR;
- }
-
EarlyTerminatingCollector(final Collector delegate, int maxCountHits) {
this.maxCountHits = maxCountHits;
- this.delegate = (delegate == null) ? NOOP_COLLECTOR : delegate;
+ this.delegate = Objects.requireNonNull(delegate);
}
- public void reset() {
- count = 0;
- }
public int count() {
return count;
}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/NoopCollector.java b/core/src/main/java/org/elasticsearch/common/lucene/search/NoopCollector.java
deleted file mode 100644
index 99845ea18d6..00000000000
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/NoopCollector.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.lucene.search;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SimpleCollector;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class NoopCollector extends SimpleCollector {
-
- public static final NoopCollector NOOP_COLLECTOR = new NoopCollector();
-
- @Override
- public void setScorer(Scorer scorer) throws IOException {
- }
-
- @Override
- public void collect(int doc) throws IOException {
- }
-
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- }
-
- @Override
- public boolean needsScores() {
- return false;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
index 77eb218a0b6..55586d8fbc2 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
@@ -33,10 +33,24 @@ import java.util.concurrent.ConcurrentMap;
/** Utility class to resolve the Lucene doc ID and version for a given uid. */
public class Versions {
- public static final long MATCH_ANY = -3L; // Version was not specified by the user
+ /** used to indicate the write operation should succeed regardless of current version **/
+ public static final long MATCH_ANY = -3L;
+
+ /** indicates that the current document was not found in lucene and in the version map */
public static final long NOT_FOUND = -1L;
+
+ /**
+ * used when the document is old and doesn't contain any version information in the index
+ * see {@link PerThreadIDAndVersionLookup#lookup(org.apache.lucene.util.BytesRef)}
+ */
public static final long NOT_SET = -2L;
+ /**
+ * used to indicate that the write operation should be executed if the document is currently deleted
+ * i.e., not found in the index and/or found as deleted (with version) in the version map
+ */
+ public static final long MATCH_DELETED = -4L;
+
// TODO: is there somewhere else we can store these?
private static final ConcurrentMap> lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
diff --git a/core/src/main/java/org/elasticsearch/common/network/InetAddresses.java b/core/src/main/java/org/elasticsearch/common/network/InetAddresses.java
new file mode 100644
index 00000000000..4d3d140ae63
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/network/InetAddresses.java
@@ -0,0 +1,357 @@
+/*
+ * Copyright (C) 2008 The Guava Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.common.network;
+
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Locale;
+
+public class InetAddresses {
+ private static int IPV4_PART_COUNT = 4;
+ private static int IPV6_PART_COUNT = 8;
+
+ public static boolean isInetAddress(String ipString) {
+ return ipStringToBytes(ipString) != null;
+ }
+
+ private static byte[] ipStringToBytes(String ipString) {
+ // Make a first pass to categorize the characters in this string.
+ boolean hasColon = false;
+ boolean hasDot = false;
+ for (int i = 0; i < ipString.length(); i++) {
+ char c = ipString.charAt(i);
+ if (c == '.') {
+ hasDot = true;
+ } else if (c == ':') {
+ if (hasDot) {
+ return null; // Colons must not appear after dots.
+ }
+ hasColon = true;
+ } else if (Character.digit(c, 16) == -1) {
+ return null; // Everything else must be a decimal or hex digit.
+ }
+ }
+
+ // Now decide which address family to parse.
+ if (hasColon) {
+ if (hasDot) {
+ ipString = convertDottedQuadToHex(ipString);
+ if (ipString == null) {
+ return null;
+ }
+ }
+ return textToNumericFormatV6(ipString);
+ } else if (hasDot) {
+ return textToNumericFormatV4(ipString);
+ }
+ return null;
+ }
+
+ private static String convertDottedQuadToHex(String ipString) {
+ int lastColon = ipString.lastIndexOf(':');
+ String initialPart = ipString.substring(0, lastColon + 1);
+ String dottedQuad = ipString.substring(lastColon + 1);
+ byte[] quad = textToNumericFormatV4(dottedQuad);
+ if (quad == null) {
+ return null;
+ }
+ String penultimate = Integer.toHexString(((quad[0] & 0xff) << 8) | (quad[1] & 0xff));
+ String ultimate = Integer.toHexString(((quad[2] & 0xff) << 8) | (quad[3] & 0xff));
+ return initialPart + penultimate + ":" + ultimate;
+ }
+
+ private static byte[] textToNumericFormatV4(String ipString) {
+ String[] address = ipString.split("\\.", IPV4_PART_COUNT + 1);
+ if (address.length != IPV4_PART_COUNT) {
+ return null;
+ }
+
+ byte[] bytes = new byte[IPV4_PART_COUNT];
+ try {
+ for (int i = 0; i < bytes.length; i++) {
+ bytes[i] = parseOctet(address[i]);
+ }
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+
+ return bytes;
+ }
+
+ private static byte parseOctet(String ipPart) {
+ // Note: we already verified that this string contains only hex digits.
+ int octet = Integer.parseInt(ipPart);
+ // Disallow leading zeroes, because no clear standard exists on
+ // whether these should be interpreted as decimal or octal.
+ if (octet > 255 || (ipPart.startsWith("0") && ipPart.length() > 1)) {
+ throw new NumberFormatException();
+ }
+ return (byte) octet;
+ }
+
+ private static byte[] textToNumericFormatV6(String ipString) {
+ // An address can have [2..8] colons, and N colons make N+1 parts.
+ String[] parts = ipString.split(":", IPV6_PART_COUNT + 2);
+ if (parts.length < 3 || parts.length > IPV6_PART_COUNT + 1) {
+ return null;
+ }
+
+ // Disregarding the endpoints, find "::" with nothing in between.
+ // This indicates that a run of zeroes has been skipped.
+ int skipIndex = -1;
+ for (int i = 1; i < parts.length - 1; i++) {
+ if (parts[i].length() == 0) {
+ if (skipIndex >= 0) {
+ return null; // Can't have more than one ::
+ }
+ skipIndex = i;
+ }
+ }
+
+ int partsHi; // Number of parts to copy from above/before the "::"
+ int partsLo; // Number of parts to copy from below/after the "::"
+ if (skipIndex >= 0) {
+ // If we found a "::", then check if it also covers the endpoints.
+ partsHi = skipIndex;
+ partsLo = parts.length - skipIndex - 1;
+ if (parts[0].length() == 0 && --partsHi != 0) {
+ return null; // ^: requires ^::
+ }
+ if (parts[parts.length - 1].length() == 0 && --partsLo != 0) {
+ return null; // :$ requires ::$
+ }
+ } else {
+ // Otherwise, allocate the entire address to partsHi. The endpoints
+ // could still be empty, but parseHextet() will check for that.
+ partsHi = parts.length;
+ partsLo = 0;
+ }
+
+ // If we found a ::, then we must have skipped at least one part.
+ // Otherwise, we must have exactly the right number of parts.
+ int partsSkipped = IPV6_PART_COUNT - (partsHi + partsLo);
+ if (!(skipIndex >= 0 ? partsSkipped >= 1 : partsSkipped == 0)) {
+ return null;
+ }
+
+ // Now parse the hextets into a byte array.
+ ByteBuffer rawBytes = ByteBuffer.allocate(2 * IPV6_PART_COUNT);
+ try {
+ for (int i = 0; i < partsHi; i++) {
+ rawBytes.putShort(parseHextet(parts[i]));
+ }
+ for (int i = 0; i < partsSkipped; i++) {
+ rawBytes.putShort((short) 0);
+ }
+ for (int i = partsLo; i > 0; i--) {
+ rawBytes.putShort(parseHextet(parts[parts.length - i]));
+ }
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+ return rawBytes.array();
+ }
+
+ private static short parseHextet(String ipPart) {
+ // Note: we already verified that this string contains only hex digits.
+ int hextet = Integer.parseInt(ipPart, 16);
+ if (hextet > 0xffff) {
+ throw new NumberFormatException();
+ }
+ return (short) hextet;
+ }
+
+ /**
+ * Returns the string representation of an {@link InetAddress} suitable
+ * for inclusion in a URI.
+ *
+ * For IPv4 addresses, this is identical to
+ * {@link InetAddress#getHostAddress()}, but for IPv6 addresses it
+ * compresses zeroes and surrounds the text with square brackets; for example
+ * {@code "[2001:db8::1]"}.
+ *
+ *
Per section 3.2.2 of
+ * http://tools.ietf.org/html/rfc3986 ,
+ * a URI containing an IPv6 string literal is of the form
+ * {@code "http://[2001:db8::1]:8888/index.html"}.
+ *
+ *
Use of either {@link InetAddresses#toAddrString},
+ * {@link InetAddress#getHostAddress()}, or this method is recommended over
+ * {@link InetAddress#toString()} when an IP address string literal is
+ * desired. This is because {@link InetAddress#toString()} prints the
+ * hostname and the IP address string joined by a "/".
+ *
+ * @param ip {@link InetAddress} to be converted to URI string literal
+ * @return {@code String} containing URI-safe string literal
+ */
+ public static String toUriString(InetAddress ip) {
+ if (ip instanceof Inet6Address) {
+ return "[" + toAddrString(ip) + "]";
+ }
+ return toAddrString(ip);
+ }
+
+ /**
+ * Returns the string representation of an {@link InetAddress}.
+ *
+ *
For IPv4 addresses, this is identical to
+ * {@link InetAddress#getHostAddress()}, but for IPv6 addresses, the output
+ * follows RFC 5952
+ * section 4. The main difference is that this method uses "::" for zero
+ * compression, while Java's version uses the uncompressed form.
+ *
+ *
This method uses hexadecimal for all IPv6 addresses, including
+ * IPv4-mapped IPv6 addresses such as "::c000:201". The output does not
+ * include a Scope ID.
+ *
+ * @param ip {@link InetAddress} to be converted to an address string
+ * @return {@code String} containing the text-formatted IP address
+ * @since 10.0
+ */
+ public static String toAddrString(InetAddress ip) {
+ if (ip == null) {
+ throw new NullPointerException("ip");
+ }
+ if (ip instanceof Inet4Address) {
+ // For IPv4, Java's formatting is good enough.
+ byte[] bytes = ip.getAddress();
+ return (bytes[0] & 0xff) + "." + (bytes[1] & 0xff) + "." + (bytes[2] & 0xff) + "." + (bytes[3] & 0xff);
+ }
+ if (!(ip instanceof Inet6Address)) {
+ throw new IllegalArgumentException("ip");
+ }
+ byte[] bytes = ip.getAddress();
+ int[] hextets = new int[IPV6_PART_COUNT];
+ for (int i = 0; i < hextets.length; i++) {
+ hextets[i] = (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
+ }
+ compressLongestRunOfZeroes(hextets);
+ return hextetsToIPv6String(hextets);
+ }
+
+ /**
+ * Identify and mark the longest run of zeroes in an IPv6 address.
+ *
+ *
Only runs of two or more hextets are considered. In case of a tie, the
+ * leftmost run wins. If a qualifying run is found, its hextets are replaced
+ * by the sentinel value -1.
+ *
+ * @param hextets {@code int[]} mutable array of eight 16-bit hextets
+ */
+ private static void compressLongestRunOfZeroes(int[] hextets) {
+ int bestRunStart = -1;
+ int bestRunLength = -1;
+ int runStart = -1;
+ for (int i = 0; i < hextets.length + 1; i++) {
+ if (i < hextets.length && hextets[i] == 0) {
+ if (runStart < 0) {
+ runStart = i;
+ }
+ } else if (runStart >= 0) {
+ int runLength = i - runStart;
+ if (runLength > bestRunLength) {
+ bestRunStart = runStart;
+ bestRunLength = runLength;
+ }
+ runStart = -1;
+ }
+ }
+ if (bestRunLength >= 2) {
+ Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1);
+ }
+ }
+
+ /**
+ * Convert a list of hextets into a human-readable IPv6 address.
+ *
+ *
In order for "::" compression to work, the input should contain negative
+ * sentinel values in place of the elided zeroes.
+ *
+ * @param hextets {@code int[]} array of eight 16-bit hextets, or -1s
+ */
+ private static String hextetsToIPv6String(int[] hextets) {
+ /*
+ * While scanning the array, handle these state transitions:
+ * start->num => "num" start->gap => "::"
+ * num->num => ":num" num->gap => "::"
+ * gap->num => "num" gap->gap => ""
+ */
+ StringBuilder buf = new StringBuilder(39);
+ boolean lastWasNumber = false;
+ for (int i = 0; i < hextets.length; i++) {
+ boolean thisIsNumber = hextets[i] >= 0;
+ if (thisIsNumber) {
+ if (lastWasNumber) {
+ buf.append(':');
+ }
+ buf.append(Integer.toHexString(hextets[i]));
+ } else {
+ if (i == 0 || lastWasNumber) {
+ buf.append("::");
+ }
+ }
+ lastWasNumber = thisIsNumber;
+ }
+ return buf.toString();
+ }
+
+ /**
+ * Returns the {@link InetAddress} having the given string representation.
+ *
+ *
This deliberately avoids all nameservice lookups (e.g. no DNS).
+ *
+ * @param ipString {@code String} containing an IPv4 or IPv6 string literal, e.g.
+ * {@code "192.168.0.1"} or {@code "2001:db8::1"}
+ * @return {@link InetAddress} representing the argument
+ * @throws IllegalArgumentException if the argument is not a valid IP string literal
+ */
+ public static InetAddress forString(String ipString) {
+ byte[] addr = ipStringToBytes(ipString);
+
+ // The argument was malformed, i.e. not an IP string literal.
+ if (addr == null) {
+ throw new IllegalArgumentException(String.format(Locale.ROOT, "'%s' is not an IP string literal.", ipString));
+ }
+
+ return bytesToInetAddress(addr);
+ }
+
+ /**
+ * Convert a byte array into an InetAddress.
+ *
+ * {@link InetAddress#getByAddress} is documented as throwing a checked
+ * exception "if IP address is of illegal length." We replace it with
+ * an unchecked exception, for use by callers who already know that addr
+ * is an array of length 4 or 16.
+ *
+ * @param addr the raw 4-byte or 16-byte IP address in big-endian order
+ * @return an InetAddress object created from the raw IP address
+ */
+ private static InetAddress bytesToInetAddress(byte[] addr) {
+ try {
+ return InetAddress.getByAddress(addr);
+ } catch (UnknownHostException e) {
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkAddress.java b/core/src/main/java/org/elasticsearch/common/network/NetworkAddress.java
index 91eda6bb624..3dcaeeb1f0e 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkAddress.java
@@ -19,8 +19,6 @@
package org.elasticsearch.common.network;
-import com.google.common.net.InetAddresses;
-
import org.elasticsearch.common.SuppressForbidden;
import java.net.Inet6Address;
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
index 8eff70e7bd8..cd46d1416f4 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.unit.TimeValue;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
@@ -73,7 +72,7 @@ public class NetworkService extends AbstractComponent {
/**
* Resolves a custom value handling, return null if can't handle it.
*/
- InetAddress[] resolveIfPossible(String value);
+ InetAddress[] resolveIfPossible(String value) throws IOException;
}
private final List customNameResolvers = new CopyOnWriteArrayList<>();
@@ -162,7 +161,7 @@ public class NetworkService extends AbstractComponent {
return address;
}
- private InetAddress[] resolveInetAddress(String host) throws UnknownHostException, IOException {
+ private InetAddress[] resolveInetAddress(String host) throws IOException {
if ((host.startsWith("#") && host.endsWith("#")) || (host.startsWith("_") && host.endsWith("_"))) {
host = host.substring(1, host.length() - 1);
// allow custom resolvers to have special names
diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
index 243c9443f08..a36c37b22e9 100644
--- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
@@ -23,16 +23,12 @@ import com.carrotsearch.hppc.DoubleArrayList;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.LongArrayList;
import com.carrotsearch.hppc.ObjectArrayList;
-import com.google.common.collect.Iterators;
-
import org.apache.lucene.util.*;
import java.util.*;
/** Collections-related utility methods. */
-public enum CollectionUtils {
- CollectionUtils;
-
+public class CollectionUtils {
public static void sort(LongArrayList list) {
sort(list.buffer, list.size());
}
@@ -367,13 +363,6 @@ public enum CollectionUtils {
}
- /**
- * Combines multiple iterators into a single iterator.
- */
- public static Iterator concat(Iterator... iterators) {
- return Iterators.concat(iterators);
- }
-
public static ArrayList iterableAsArrayList(Iterable elements) {
if (elements == null) {
throw new NullPointerException("elements");
diff --git a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
index fe8428425e2..0d5c3cb12c1 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.elasticsearch.index;
import org.elasticsearch.common.Nullable;
@@ -34,6 +35,7 @@ import org.elasticsearch.index.termvectors.TermVectorsService;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
+import org.elasticsearch.indices.memory.IndexingMemoryController;
import org.elasticsearch.threadpool.ThreadPool;
/**
@@ -58,9 +60,10 @@ public final class IndexServicesProvider {
private final EngineFactory factory;
private final BigArrays bigArrays;
private final IndexSearcherWrapper indexSearcherWrapper;
+ private final IndexingMemoryController indexingMemoryController;
@Inject
- public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper) {
+ public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper, IndexingMemoryController indexingMemoryController) {
this.indicesLifecycle = indicesLifecycle;
this.threadPool = threadPool;
this.mapperService = mapperService;
@@ -76,6 +79,7 @@ public final class IndexServicesProvider {
this.factory = factory;
this.bigArrays = bigArrays;
this.indexSearcherWrapper = indexSearcherWrapper;
+ this.indexingMemoryController = indexingMemoryController;
}
public IndicesLifecycle getIndicesLifecycle() {
@@ -134,5 +138,11 @@ public final class IndexServicesProvider {
return bigArrays;
}
- public IndexSearcherWrapper getIndexSearcherWrapper() { return indexSearcherWrapper; }
+ public IndexSearcherWrapper getIndexSearcherWrapper() {
+ return indexSearcherWrapper;
+ }
+
+ public IndexingMemoryController getIndexingMemoryController() {
+ return indexingMemoryController;
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/VersionType.java b/core/src/main/java/org/elasticsearch/index/VersionType.java
index a5d8cae2453..b8f998b9704 100644
--- a/core/src/main/java/org/elasticsearch/index/VersionType.java
+++ b/core/src/main/java/org/elasticsearch/index/VersionType.java
@@ -31,24 +31,37 @@ import java.io.IOException;
public enum VersionType implements Writeable {
INTERNAL((byte) 0) {
@Override
- public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
- return isVersionConflict(currentVersion, expectedVersion);
+ public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+ return isVersionConflict(currentVersion, expectedVersion, deleted);
+ }
+
+ @Override
+ public String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+ if (expectedVersion == Versions.MATCH_DELETED) {
+ return "document already exists (current version [" + currentVersion + "])";
+ }
+ return "current version [" + currentVersion + "] is different than the one provided [" + expectedVersion + "]";
}
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
- return isVersionConflict(currentVersion, expectedVersion);
+ return isVersionConflict(currentVersion, expectedVersion, false);
}
- private boolean isVersionConflict(long currentVersion, long expectedVersion) {
+ @Override
+ public String explainConflictForReads(long currentVersion, long expectedVersion) {
+ return "current version [" + currentVersion + "] is different than the one provided [" + expectedVersion + "]";
+ }
+
+ private boolean isVersionConflict(long currentVersion, long expectedVersion, boolean deleted) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
if (expectedVersion == Versions.MATCH_ANY) {
return false;
}
- if (currentVersion == Versions.NOT_FOUND) {
- return true;
+ if (expectedVersion == Versions.MATCH_DELETED) {
+ return deleted == false;
}
if (currentVersion != expectedVersion) {
return true;
@@ -63,8 +76,7 @@ public enum VersionType implements Writeable {
@Override
public boolean validateVersionForWrites(long version) {
- // not allowing Versions.NOT_FOUND as it is not a valid input value.
- return version > 0L || version == Versions.MATCH_ANY;
+ return version > 0L || version == Versions.MATCH_ANY || version == Versions.MATCH_DELETED;
}
@Override
@@ -82,7 +94,7 @@ public enum VersionType implements Writeable {
},
EXTERNAL((byte) 1) {
@Override
- public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
+ public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@@ -98,6 +110,11 @@ public enum VersionType implements Writeable {
return false;
}
+ @Override
+ public String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+ return "current version [" + currentVersion + "] is higher or equal to the one provided [" + expectedVersion + "]";
+ }
+
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
@@ -115,6 +132,11 @@ public enum VersionType implements Writeable {
return false;
}
+ @Override
+ public String explainConflictForReads(long currentVersion, long expectedVersion) {
+ return "current version [" + currentVersion + "] is different than the one provided [" + expectedVersion + "]";
+ }
+
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
@@ -133,7 +155,7 @@ public enum VersionType implements Writeable {
},
EXTERNAL_GTE((byte) 2) {
@Override
- public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
+ public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@@ -149,6 +171,11 @@ public enum VersionType implements Writeable {
return false;
}
+ @Override
+ public String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+ return "current version [" + currentVersion + "] is higher than the one provided [" + expectedVersion + "]";
+ }
+
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
if (currentVersion == Versions.NOT_SET) {
@@ -166,6 +193,11 @@ public enum VersionType implements Writeable {
return false;
}
+ @Override
+ public String explainConflictForReads(long currentVersion, long expectedVersion) {
+ return "current version [" + currentVersion + "] is different than the one provided [" + expectedVersion + "]";
+ }
+
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
@@ -187,7 +219,7 @@ public enum VersionType implements Writeable {
*/
FORCE((byte) 3) {
@Override
- public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion) {
+ public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
if (currentVersion == Versions.NOT_SET) {
return false;
}
@@ -195,16 +227,26 @@ public enum VersionType implements Writeable {
return false;
}
if (expectedVersion == Versions.MATCH_ANY) {
- return true;
+ throw new IllegalStateException("you must specify a version when use VersionType.FORCE");
}
return false;
}
+ @Override
+ public String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
+ throw new AssertionError("VersionType.FORCE should never result in a write conflict");
+ }
+
@Override
public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
return false;
}
+ @Override
+ public String explainConflictForReads(long currentVersion, long expectedVersion) {
+ throw new AssertionError("VersionType.FORCE should never result in a read conflict");
+ }
+
@Override
public long updateVersion(long currentVersion, long expectedVersion) {
return expectedVersion;
@@ -237,17 +279,46 @@ public enum VersionType implements Writeable {
/**
* Checks whether the current version conflicts with the expected version, based on the current version type.
*
+ * @param currentVersion the current version for the document
+ * @param expectedVersion the version specified for the write operation
+ * @param deleted true if the document is currently deleted (note that #currentVersion will typically be
+ * {@link Versions#NOT_FOUND}, but may be something else if the document was recently deleted
* @return true if versions conflict false o.w.
*/
- public abstract boolean isVersionConflictForWrites(long currentVersion, long expectedVersion);
+ public abstract boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted);
+
+
+ /**
+ * Returns a human readable explanation for a version conflict on write.
+ *
+ * Note that this method is only called if {@link #isVersionConflictForWrites(long, long, boolean)} returns true;
+ *
+ * @param currentVersion the current version for the document
+ * @param expectedVersion the version specified for the write operation
+ * @param deleted true if the document is currently deleted (note that #currentVersion will typically be
+ * {@link Versions#NOT_FOUND}, but may be something else if the document was recently deleted
+ */
+ public abstract String explainConflictForWrites(long currentVersion, long expectedVersion, boolean deleted);
/**
* Checks whether the current version conflicts with the expected version, based on the current version type.
*
+ * @param currentVersion the current version for the document
+ * @param expectedVersion the version specified for the read operation
* @return true if versions conflict false o.w.
*/
public abstract boolean isVersionConflictForReads(long currentVersion, long expectedVersion);
+ /**
+ * Returns a human readable explanation for a version conflict on read.
+ *
+ * Note that this method is only called if {@link #isVersionConflictForReads(long, long)} returns true;
+ *
+ * @param currentVersion the current version for the document
+ * @param expectedVersion the version specified for the read operation
+ */
+ public abstract String explainConflictForReads(long currentVersion, long expectedVersion);
+
/**
* Returns the new version for a document, based on its current one and the specified in the request
*
diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 30c0905dc0f..f2b7ba8e131 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -19,11 +19,6 @@
package org.elasticsearch.index.cache.bitset;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -38,6 +33,10 @@ import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.BitSet;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.cache.RemovalListener;
+import org.elasticsearch.common.cache.RemovalNotification;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.settings.Settings;
@@ -58,10 +57,11 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
import java.util.HashSet;
-import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
/**
* This is a cache for {@link BitDocIdSet} based filters and is unbounded by size or time.
@@ -94,10 +94,11 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
public BitsetFilterCache(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
this.loadRandomAccessFiltersEagerly = indexSettings.getAsBoolean(LOAD_RANDOM_ACCESS_FILTERS_EAGERLY, true);
- this.loadedFilters = CacheBuilder.newBuilder().removalListener(this).build();
+ this.loadedFilters = CacheBuilder.>builder().removalListener(this).build();
this.warmer = new BitSetProducerWarmer();
}
+
@Inject(optional = true)
public void setIndicesWarmer(IndicesWarmer indicesWarmer) {
this.indicesWarmer = indicesWarmer;
@@ -144,14 +145,12 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext context) throws IOException, ExecutionException {
final Object coreCacheReader = context.reader().getCoreCacheKey();
final ShardId shardId = ShardUtils.extractShardId(context.reader());
- Cache filterToFbs = loadedFilters.get(coreCacheReader, new Callable>() {
- @Override
- public Cache call() throws Exception {
- context.reader().addCoreClosedListener(BitsetFilterCache.this);
- return CacheBuilder.newBuilder().build();
- }
+ Cache filterToFbs = loadedFilters.computeIfAbsent(coreCacheReader, key -> {
+ context.reader().addCoreClosedListener(BitsetFilterCache.this);
+ return CacheBuilder.builder().build();
});
- return filterToFbs.get(query, () -> {
+
+ return filterToFbs.computeIfAbsent(query, key -> {
final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
final IndexSearcher searcher = new IndexSearcher(topLevelContext);
searcher.setQueryCache(null);
@@ -172,8 +171,7 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
@Override
public void onRemoval(RemovalNotification> notification) {
- Object key = notification.getKey();
- if (key == null) {
+ if (notification.getKey() == null) {
return;
}
@@ -182,7 +180,7 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
return;
}
- for (Value value : valueCache.asMap().values()) {
+ for (Value value : valueCache.values()) {
listener.onRemoval(value.shardId, value.bitset);
// if null then this means the shard has already been removed and the stats are 0 anyway for the shard this key belongs to
}
diff --git a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
index ef82e73dc4d..0f594d2faca 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
@@ -19,10 +19,8 @@
package org.elasticsearch.index.cache.request;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.cache.RemovalListener;
+import org.elasticsearch.common.cache.RemovalNotification;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
@@ -61,7 +59,7 @@ public class ShardRequestCache extends AbstractIndexShardComponent implements Re
@Override
public void onRemoval(RemovalNotification removalNotification) {
- if (removalNotification.wasEvicted()) {
+ if (removalNotification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED) {
evictionsMetric.inc();
}
long dec = 0;
diff --git a/core/src/main/java/org/elasticsearch/index/codec/postingsformat/BloomFilterPostingsFormat.java b/core/src/main/java/org/elasticsearch/index/codec/postingsformat/BloomFilterPostingsFormat.java
deleted file mode 100644
index 71a52a7fbd1..00000000000
--- a/core/src/main/java/org/elasticsearch/index/codec/postingsformat/BloomFilterPostingsFormat.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.codec.postingsformat;
-
-import org.apache.lucene.codecs.*;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.*;
-import org.apache.lucene.util.*;
-import org.elasticsearch.common.util.BloomFilter;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.Map.Entry;
-
-/**
- *
- * A {@link PostingsFormat} useful for low doc-frequency fields such as primary
- * keys. Bloom filters are maintained in a ".blm" file which offers "fast-fail"
- * for reads in segments known to have no record of the key. A choice of
- * delegate PostingsFormat is used to record all other Postings data.
- *
- *
- * This is a special bloom filter version, based on {@link org.elasticsearch.common.util.BloomFilter} and inspired
- * by Lucene {@code org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat}.
- * @deprecated only for reading old segments
- */
-@Deprecated
-public class BloomFilterPostingsFormat extends PostingsFormat {
-
- public static final String BLOOM_CODEC_NAME = "XBloomFilter"; // the Lucene one is named BloomFilter
- public static final int BLOOM_CODEC_VERSION = 1;
- public static final int BLOOM_CODEC_VERSION_CHECKSUM = 2;
- public static final int BLOOM_CODEC_VERSION_CURRENT = BLOOM_CODEC_VERSION_CHECKSUM;
-
- /**
- * Extension of Bloom Filters file
- */
- static final String BLOOM_EXTENSION = "blm";
-
- private BloomFilter.Factory bloomFilterFactory = BloomFilter.Factory.DEFAULT;
- private PostingsFormat delegatePostingsFormat;
-
- /**
- * Creates Bloom filters for a selection of fields created in the index. This
- * is recorded as a set of Bitsets held as a segment summary in an additional
- * "blm" file. This PostingsFormat delegates to a choice of delegate
- * PostingsFormat for encoding all other postings data.
- *
- * @param delegatePostingsFormat The PostingsFormat that records all the non-bloom filter data i.e.
- * postings info.
- * @param bloomFilterFactory The {@link org.elasticsearch.common.util.BloomFilter.Factory} responsible for sizing BloomFilters
- * appropriately
- */
- public BloomFilterPostingsFormat(PostingsFormat delegatePostingsFormat,
- BloomFilter.Factory bloomFilterFactory) {
- super(BLOOM_CODEC_NAME);
- this.delegatePostingsFormat = delegatePostingsFormat;
- this.bloomFilterFactory = bloomFilterFactory;
- }
-
- // Used only by core Lucene at read-time via Service Provider instantiation -
- // do not use at Write-time in application code.
- public BloomFilterPostingsFormat() {
- super(BLOOM_CODEC_NAME);
- }
-
- @Override
- public BloomFilteredFieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("this codec can only be used for reading");
- }
-
- @Override
- public BloomFilteredFieldsProducer fieldsProducer(SegmentReadState state)
- throws IOException {
- return new BloomFilteredFieldsProducer(state);
- }
-
- public PostingsFormat getDelegate() {
- return delegatePostingsFormat;
- }
-
- private final class LazyBloomLoader implements Accountable {
- private final long offset;
- private final IndexInput indexInput;
- private BloomFilter filter;
-
- private LazyBloomLoader(long offset, IndexInput origial) {
- this.offset = offset;
- this.indexInput = origial.clone();
- }
-
- synchronized BloomFilter get() throws IOException {
- if (filter == null) {
- try (final IndexInput input = indexInput) {
- input.seek(offset);
- this.filter = BloomFilter.deserialize(input);
- }
- }
- return filter;
- }
-
- @Override
- public long ramBytesUsed() {
- return filter == null ? 0l : filter.getSizeInBytes();
- }
-
- @Override
- public Collection getChildResources() {
- return Collections.singleton(Accountables.namedAccountable("bloom", ramBytesUsed()));
- }
- }
-
- public final class BloomFilteredFieldsProducer extends FieldsProducer {
- private FieldsProducer delegateFieldsProducer;
- HashMap bloomsByFieldName = new HashMap<>();
- private final int version;
- private final IndexInput data;
-
- // for internal use only
- FieldsProducer getDelegate() {
- return delegateFieldsProducer;
- }
-
- public BloomFilteredFieldsProducer(SegmentReadState state)
- throws IOException {
-
- final String bloomFileName = IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
- final Directory directory = state.directory;
- IndexInput dataInput = directory.openInput(bloomFileName, state.context);
- try {
- ChecksumIndexInput bloomIn = new BufferedChecksumIndexInput(dataInput.clone());
- version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION,
- BLOOM_CODEC_VERSION_CURRENT);
- // // Load the hash function used in the BloomFilter
- // hashFunction = HashFunction.forName(bloomIn.readString());
- // Load the delegate postings format
- final String delegatePostings = bloomIn.readString();
- this.delegateFieldsProducer = PostingsFormat.forName(delegatePostings)
- .fieldsProducer(state);
- this.data = dataInput;
- dataInput = null; // null it out such that we don't close it
- } finally {
- IOUtils.closeWhileHandlingException(dataInput);
- }
- }
-
- @Override
- public Iterator iterator() {
- return delegateFieldsProducer.iterator();
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(data, delegateFieldsProducer);
- }
-
- @Override
- public Terms terms(String field) throws IOException {
- LazyBloomLoader filter = bloomsByFieldName.get(field);
- if (filter == null) {
- return delegateFieldsProducer.terms(field);
- } else {
- Terms result = delegateFieldsProducer.terms(field);
- if (result == null) {
- return null;
- }
- return new BloomFilteredTerms(result, filter.get());
- }
- }
-
- @Override
- public int size() {
- return delegateFieldsProducer.size();
- }
-
- @Override
- public long ramBytesUsed() {
- long size = delegateFieldsProducer.ramBytesUsed();
- for (LazyBloomLoader bloomFilter : bloomsByFieldName.values()) {
- size += bloomFilter.ramBytesUsed();
- }
- return size;
- }
-
- @Override
- public Collection getChildResources() {
- List resources = new ArrayList<>();
- resources.addAll(Accountables.namedAccountables("field", bloomsByFieldName));
- if (delegateFieldsProducer != null) {
- resources.add(Accountables.namedAccountable("delegate", delegateFieldsProducer));
- }
- return Collections.unmodifiableList(resources);
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- delegateFieldsProducer.checkIntegrity();
- if (version >= BLOOM_CODEC_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(data);
- }
- }
-
- @Override
- public FieldsProducer getMergeInstance() throws IOException {
- return delegateFieldsProducer.getMergeInstance();
- }
- }
-
- public static final class BloomFilteredTerms extends FilterLeafReader.FilterTerms {
- private BloomFilter filter;
-
- public BloomFilteredTerms(Terms terms, BloomFilter filter) {
- super(terms);
- this.filter = filter;
- }
-
- public BloomFilter getFilter() {
- return filter;
- }
-
- @Override
- public TermsEnum iterator() throws IOException {
- return new BloomFilteredTermsEnum(this.in, filter);
- }
- }
-
- static final class BloomFilteredTermsEnum extends TermsEnum {
-
- private Terms delegateTerms;
- private TermsEnum delegateTermsEnum;
- private BloomFilter filter;
-
- public BloomFilteredTermsEnum(Terms other, BloomFilter filter) {
- this.delegateTerms = other;
- this.filter = filter;
- }
-
- void reset(Terms others) {
- this.delegateTermsEnum = null;
- this.delegateTerms = others;
- }
-
- private TermsEnum getDelegate() throws IOException {
- if (delegateTermsEnum == null) {
- /* pull the iterator only if we really need it -
- * this can be a relatively heavy operation depending on the
- * delegate postings format and they underlying directory
- * (clone IndexInput) */
- delegateTermsEnum = delegateTerms.iterator();
- }
- return delegateTermsEnum;
- }
-
- @Override
- public final BytesRef next() throws IOException {
- return getDelegate().next();
- }
-
- @Override
- public final boolean seekExact(BytesRef text)
- throws IOException {
- // The magical fail-fast speed up that is the entire point of all of
- // this code - save a disk seek if there is a match on an in-memory
- // structure
- // that may occasionally give a false positive but guaranteed no false
- // negatives
- if (!filter.mightContain(text)) {
- return false;
- }
- return getDelegate().seekExact(text);
- }
-
- @Override
- public final SeekStatus seekCeil(BytesRef text)
- throws IOException {
- return getDelegate().seekCeil(text);
- }
-
- @Override
- public final void seekExact(long ord) throws IOException {
- getDelegate().seekExact(ord);
- }
-
- @Override
- public final BytesRef term() throws IOException {
- return getDelegate().term();
- }
-
- @Override
- public final long ord() throws IOException {
- return getDelegate().ord();
- }
-
- @Override
- public final int docFreq() throws IOException {
- return getDelegate().docFreq();
- }
-
- @Override
- public final long totalTermFreq() throws IOException {
- return getDelegate().totalTermFreq();
- }
-
-
- @Override
- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
- return getDelegate().postings(reuse, flags);
- }
- }
-
- // TODO: would be great to move this out to test code, but the interaction between es090 and bloom is complex
- // at least it is not accessible via SPI
- public final class BloomFilteredFieldsConsumer extends FieldsConsumer {
- private final FieldsConsumer delegateFieldsConsumer;
- private final Map bloomFilters = new HashMap<>();
- private final SegmentWriteState state;
- private boolean closed = false;
-
- // private PostingsFormat delegatePostingsFormat;
-
- public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
- SegmentWriteState state, PostingsFormat delegatePostingsFormat) {
- this.delegateFieldsConsumer = fieldsConsumer;
- // this.delegatePostingsFormat=delegatePostingsFormat;
- this.state = state;
- }
-
- // for internal use only
- public FieldsConsumer getDelegate() {
- return delegateFieldsConsumer;
- }
-
-
- @Override
- public void write(Fields fields) throws IOException {
-
- // Delegate must write first: it may have opened files
- // on creating the class
- // (e.g. Lucene41PostingsConsumer), and write() will
- // close them; alternatively, if we delayed pulling
- // the fields consumer until here, we could do it
- // afterwards:
- delegateFieldsConsumer.write(fields);
-
- for(String field : fields) {
- Terms terms = fields.terms(field);
- if (terms == null) {
- continue;
- }
- FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
- TermsEnum termsEnum = terms.iterator();
-
- BloomFilter bloomFilter = null;
-
- PostingsEnum postings = null;
- while (true) {
- BytesRef term = termsEnum.next();
- if (term == null) {
- break;
- }
- if (bloomFilter == null) {
- bloomFilter = bloomFilterFactory.createFilter(state.segmentInfo.maxDoc());
- assert bloomFilters.containsKey(field) == false;
- bloomFilters.put(fieldInfo, bloomFilter);
- }
- // Make sure there's at least one doc for this term:
- postings = termsEnum.postings(postings, 0);
- if (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
- bloomFilter.put(term);
- }
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
- delegateFieldsConsumer.close();
- // Now we are done accumulating values for these fields
- List> nonSaturatedBlooms = new ArrayList<>();
-
- for (Entry entry : bloomFilters.entrySet()) {
- nonSaturatedBlooms.add(entry);
- }
- String bloomFileName = IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
- IndexOutput bloomOutput = null;
- try {
- bloomOutput = state.directory
- .createOutput(bloomFileName, state.context);
- CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME,
- BLOOM_CODEC_VERSION_CURRENT);
- // remember the name of the postings format we will delegate to
- bloomOutput.writeString(delegatePostingsFormat.getName());
-
- // First field in the output file is the number of fields+blooms saved
- bloomOutput.writeInt(nonSaturatedBlooms.size());
- for (Entry entry : nonSaturatedBlooms) {
- FieldInfo fieldInfo = entry.getKey();
- BloomFilter bloomFilter = entry.getValue();
- bloomOutput.writeInt(fieldInfo.number);
- saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo);
- }
- CodecUtil.writeFooter(bloomOutput);
- } finally {
- IOUtils.close(bloomOutput);
- }
- //We are done with large bitsets so no need to keep them hanging around
- bloomFilters.clear();
- }
-
- private void saveAppropriatelySizedBloomFilter(IndexOutput bloomOutput,
- BloomFilter bloomFilter, FieldInfo fieldInfo) throws IOException {
- BloomFilter.serilaize(bloomFilter, bloomOutput);
- }
-
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/codec/postingsformat/Elasticsearch090PostingsFormat.java b/core/src/main/java/org/elasticsearch/index/codec/postingsformat/Elasticsearch090PostingsFormat.java
deleted file mode 100644
index b6ceba8fb8a..00000000000
--- a/core/src/main/java/org/elasticsearch/index/codec/postingsformat/Elasticsearch090PostingsFormat.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.codec.postingsformat;
-
-import com.google.common.collect.Iterators;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.util.BloomFilter;
-import org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat.BloomFilteredFieldsConsumer;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.function.Predicate;
-
-/**
- * This is the old default postings format for Elasticsearch that special cases
- * the _uid field to use a bloom filter while all other fields
- * will use a {@link Lucene50PostingsFormat}. This format will reuse the underlying
- * {@link Lucene50PostingsFormat} and its files also for the _uid saving up to
- * 5 files per segment in the default case.
- *
- * @deprecated only for reading old segments
- */
-@Deprecated
-public class Elasticsearch090PostingsFormat extends PostingsFormat {
- protected final BloomFilterPostingsFormat bloomPostings;
-
- public Elasticsearch090PostingsFormat() {
- super("es090");
- Lucene50PostingsFormat delegate = new Lucene50PostingsFormat();
- assert delegate.getName().equals(Lucene.LATEST_POSTINGS_FORMAT);
- bloomPostings = new BloomFilterPostingsFormat(delegate, BloomFilter.Factory.DEFAULT);
- }
-
- public PostingsFormat getDefaultWrapped() {
- return bloomPostings.getDelegate();
- }
-
- protected static final Predicate UID_FIELD_FILTER = field -> UidFieldMapper.NAME.equals(field);
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("this codec can only be used for reading");
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- // we can just return the delegate here since we didn't record bloom filters for
- // the other fields.
- return bloomPostings.fieldsProducer(state);
- }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/CreateFailedEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/CreateFailedEngineException.java
deleted file mode 100644
index 32d9ee620c8..00000000000
--- a/core/src/main/java/org/elasticsearch/index/engine/CreateFailedEngineException.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.engine;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- *
- */
-public class CreateFailedEngineException extends EngineException {
-
- private final String type;
-
- private final String id;
-
- public CreateFailedEngineException(ShardId shardId, String type, String id, Throwable cause) {
- super(shardId, "Create failed for [" + type + "#" + id + "]", cause);
- Objects.requireNonNull(type, "type must not be null");
- Objects.requireNonNull(id, "id must not be null");
- this.type = type;
- this.id = id;
- }
-
- public CreateFailedEngineException(StreamInput in) throws IOException{
- super(in);
- type = in.readString();
- id = in.readString();
- }
-
- public String type() {
- return this.type;
- }
-
- public String id() {
- return this.id;
- }
-
- @Override
- public void writeTo(StreamOutput out) throws IOException {
- super.writeTo(out);
- out.writeString(type);
- out.writeString(id);
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/DeleteByQueryFailedEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/DeleteByQueryFailedEngineException.java
deleted file mode 100644
index 95d57c53836..00000000000
--- a/core/src/main/java/org/elasticsearch/index/engine/DeleteByQueryFailedEngineException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.engine;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-
-/** @deprecated Delete-by-query is removed in 2.0, but we keep this so translog can replay on upgrade. */
-@Deprecated
-public class DeleteByQueryFailedEngineException extends EngineException {
-
- public DeleteByQueryFailedEngineException(ShardId shardId, Engine.DeleteByQuery deleteByQuery, Throwable cause) {
- super(shardId, "Delete by query failed for [" + deleteByQuery.query() + "]", cause);
- }
-
- public DeleteByQueryFailedEngineException(StreamInput in) throws IOException{
- super(in);
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/DocumentAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/index/engine/DocumentAlreadyExistsException.java
deleted file mode 100644
index 467dd8c14c7..00000000000
--- a/core/src/main/java/org/elasticsearch/index/engine/DocumentAlreadyExistsException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.engine;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class DocumentAlreadyExistsException extends EngineException {
-
- public DocumentAlreadyExistsException(ShardId shardId, String type, String id) {
- super(shardId, "[" + type + "][" + id + "]: document already exists");
- }
-
- public DocumentAlreadyExistsException(StreamInput in) throws IOException{
- super(in);
- }
-
- @Override
- public RestStatus status() {
- return RestStatus.CONFLICT;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 1330ef05a7f..c07be064489 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -45,7 +45,6 @@ import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
@@ -60,7 +59,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
-import java.util.function.Supplier;
/**
*
@@ -144,7 +142,8 @@ public abstract class Engine implements Closeable {
return new MergeStats();
}
- /** A throttling class that can be activated, causing the
+ /**
+ * A throttling class that can be activated, causing the
* {@code acquireThrottle} method to block on a lock when throttling
* is enabled
*/
@@ -203,20 +202,15 @@ public abstract class Engine implements Closeable {
}
}
- public abstract void create(Create create) throws EngineException;
-
- public abstract boolean index(Index index) throws EngineException;
+ public abstract boolean index(Index operation) throws EngineException;
public abstract void delete(Delete delete) throws EngineException;
- /** @deprecated This was removed, but we keep this API so translog can replay any DBQs on upgrade. */
- @Deprecated
- public abstract void delete(DeleteByQuery delete) throws EngineException;
-
/**
* Attempts to do a special commit where the given syncID is put into the commit data. The attempt
* succeeds if there are not pending writes in lucene and the current point is equal to the expected one.
- * @param syncId id of this sync
+ *
+ * @param syncId id of this sync
* @param expectedCommitId the expected value of
* @return true if the sync commit was made, false o.w.
*/
@@ -243,7 +237,8 @@ public abstract class Engine implements Closeable {
if (get.versionType().isVersionConflictForReads(docIdAndVersion.version, get.version())) {
Releasables.close(searcher);
Uid uid = Uid.createUid(get.uid().text());
- throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), docIdAndVersion.version, get.version());
+ throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
+ get.versionType().explainConflictForReads(docIdAndVersion.version, get.version()));
}
}
@@ -328,7 +323,7 @@ public abstract class Engine implements Closeable {
} catch (IOException e) {
// Fall back to reading from the store if reading from the commit fails
try {
- return store. readLastCommittedSegmentsInfo();
+ return store.readLastCommittedSegmentsInfo();
} catch (IOException e2) {
e2.addSuppressed(e);
throw e2;
@@ -366,6 +361,9 @@ public abstract class Engine implements Closeable {
stats.addIndexWriterMaxMemoryInBytes(0);
}
+ /** How much heap Lucene's IndexWriter is using */
+ abstract public long indexWriterRAMBytesUsed();
+
protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
ensureOpen();
Map segments = new HashMap<>();
@@ -469,7 +467,8 @@ public abstract class Engine implements Closeable {
/**
* Flushes the state of the engine including the transaction log, clearing memory.
- * @param force if true a lucene commit is executed even if no changes need to be committed.
+ *
+ * @param force if true a lucene commit is executed even if no changes need to be committed.
* @param waitIfOngoing if true this call will block until all currently running flushes have finished.
* Otherwise this call will return without blocking.
* @return the commit Id for the resulting commit
@@ -607,62 +606,97 @@ public abstract class Engine implements Closeable {
}
}
- public static interface Operation {
- static enum Type {
- CREATE,
- INDEX,
- DELETE
- }
-
- static enum Origin {
- PRIMARY,
- REPLICA,
- RECOVERY
- }
-
- Type opType();
-
- Origin origin();
- }
-
- public static abstract class IndexingOperation implements Operation {
-
+ public static abstract class Operation {
private final Term uid;
- private final ParsedDocument doc;
private long version;
private final VersionType versionType;
private final Origin origin;
private Translog.Location location;
-
private final long startTime;
private long endTime;
- public IndexingOperation(Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
+ public Operation(Term uid, long version, VersionType versionType, Origin origin, long startTime) {
this.uid = uid;
- this.doc = doc;
this.version = version;
this.versionType = versionType;
this.origin = origin;
this.startTime = startTime;
}
- public IndexingOperation(Term uid, ParsedDocument doc) {
- this(uid, doc, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime());
+ public static enum Origin {
+ PRIMARY,
+ REPLICA,
+ RECOVERY
}
- @Override
public Origin origin() {
return this.origin;
}
- public ParsedDocument parsedDoc() {
- return this.doc;
- }
-
public Term uid() {
return this.uid;
}
+ public long version() {
+ return this.version;
+ }
+
+ public void updateVersion(long version) {
+ this.version = version;
+ }
+
+ public void setTranslogLocation(Translog.Location location) {
+ this.location = location;
+ }
+
+ public Translog.Location getTranslogLocation() {
+ return this.location;
+ }
+
+ public VersionType versionType() {
+ return this.versionType;
+ }
+
+ /**
+ * Returns operation start time in nanoseconds.
+ */
+ public long startTime() {
+ return this.startTime;
+ }
+
+ public void endTime(long endTime) {
+ this.endTime = endTime;
+ }
+
+ /**
+ * Returns operation end time in nanoseconds.
+ */
+ public long endTime() {
+ return this.endTime;
+ }
+ }
+
+ public static class Index extends Operation {
+
+ private final ParsedDocument doc;
+
+ public Index(Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
+ super(uid, version, versionType, origin, startTime);
+ this.doc = doc;
+ }
+
+ public Index(Term uid, ParsedDocument doc) {
+ this(uid, doc, Versions.MATCH_ANY);
+ }
+
+ public Index(Term uid, ParsedDocument doc, long version) {
+ this(uid, doc, version, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime());
+ }
+
+ public ParsedDocument parsedDoc() {
+ return this.doc;
+ }
+
public String type() {
return this.doc.type();
}
@@ -683,27 +717,12 @@ public abstract class Engine implements Closeable {
return this.doc.ttl();
}
- public long version() {
- return this.version;
- }
-
+ @Override
public void updateVersion(long version) {
- this.version = version;
+ super.updateVersion(version);
this.doc.version().setLongValue(version);
}
- public void setTranslogLocation(Translog.Location location) {
- this.location = location;
- }
-
- public Translog.Location getTranslogLocation() {
- return this.location;
- }
-
- public VersionType versionType() {
- return this.versionType;
- }
-
public String parent() {
return this.doc.parent();
}
@@ -715,96 +734,17 @@ public abstract class Engine implements Closeable {
public BytesReference source() {
return this.doc.source();
}
-
- /**
- * Returns operation start time in nanoseconds.
- */
- public long startTime() {
- return this.startTime;
- }
-
- public void endTime(long endTime) {
- this.endTime = endTime;
- }
-
- /**
- * Returns operation end time in nanoseconds.
- */
- public long endTime() {
- return this.endTime;
- }
-
- /**
- * Execute this operation against the provided {@link IndexShard} and
- * return whether the document was created.
- */
- public abstract boolean execute(IndexShard shard);
}
- public static final class Create extends IndexingOperation {
-
- public Create(Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
- super(uid, doc, version, versionType, origin, startTime);
- }
-
- public Create(Term uid, ParsedDocument doc) {
- super(uid, doc);
- }
-
- @Override
- public Type opType() {
- return Type.CREATE;
- }
-
- @Override
- public boolean execute(IndexShard shard) {
- shard.create(this);
- return true;
- }
- }
-
- public static final class Index extends IndexingOperation {
-
- public Index(Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
- super(uid, doc, version, versionType, origin, startTime);
- }
-
- public Index(Term uid, ParsedDocument doc) {
- super(uid, doc);
- }
-
- @Override
- public Type opType() {
- return Type.INDEX;
- }
-
- @Override
- public boolean execute(IndexShard shard) {
- return shard.index(this);
- }
- }
-
- public static class Delete implements Operation {
+ public static class Delete extends Operation {
private final String type;
private final String id;
- private final Term uid;
- private long version;
- private final VersionType versionType;
- private final Origin origin;
private boolean found;
- private final long startTime;
- private long endTime;
- private Translog.Location location;
-
public Delete(String type, String id, Term uid, long version, VersionType versionType, Origin origin, long startTime, boolean found) {
+ super(uid, version, versionType, origin, startTime);
this.type = type;
this.id = id;
- this.uid = uid;
- this.version = version;
- this.versionType = versionType;
- this.origin = origin;
- this.startTime = startTime;
this.found = found;
}
@@ -816,16 +756,6 @@ public abstract class Engine implements Closeable {
this(template.type(), template.id(), template.uid(), template.version(), versionType, template.origin(), template.startTime(), template.found());
}
- @Override
- public Type opType() {
- return Type.DELETE;
- }
-
- @Override
- public Origin origin() {
- return this.origin;
- }
-
public String type() {
return this.type;
}
@@ -834,55 +764,14 @@ public abstract class Engine implements Closeable {
return this.id;
}
- public Term uid() {
- return this.uid;
- }
-
public void updateVersion(long version, boolean found) {
- this.version = version;
+ updateVersion(version);
this.found = found;
}
- /**
- * before delete execution this is the version to be deleted. After this is the version of the "delete" transaction record.
- */
- public long version() {
- return this.version;
- }
-
- public VersionType versionType() {
- return this.versionType;
- }
-
public boolean found() {
return this.found;
}
-
- /**
- * Returns operation start time in nanoseconds.
- */
- public long startTime() {
- return this.startTime;
- }
-
- public void endTime(long endTime) {
- this.endTime = endTime;
- }
-
- /**
- * Returns operation end time in nanoseconds.
- */
- public long endTime() {
- return this.endTime;
- }
-
- public void setTranslogLocation(Translog.Location location) {
- this.location = location;
- }
-
- public Translog.Location getTranslogLocation() {
- return this.location;
- }
}
public static class DeleteByQuery {
@@ -1135,12 +1024,18 @@ public abstract class Engine implements Closeable {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
CommitId commitId = (CommitId) o;
- if (!Arrays.equals(id, commitId.id)) return false;
+ if (!Arrays.equals(id, commitId.id)) {
+ return false;
+ }
return true;
}
@@ -1151,5 +1046,6 @@ public abstract class Engine implements Closeable {
}
}
- public void onSettingsChanged() {}
+ public void onSettingsChanged() {
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index a79587e4347..fd4b5daf4ee 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -40,6 +40,7 @@ import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.TranslogConfig;
import org.elasticsearch.indices.IndicesWarmer;
+import org.elasticsearch.indices.memory.IndexingMemoryController;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.concurrent.TimeUnit;
@@ -107,8 +108,6 @@ public final class EngineConfig {
public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
public static final TimeValue DEFAULT_GC_DELETES = TimeValue.timeValueSeconds(60);
- public static final ByteSizeValue DEFAULT_INDEX_BUFFER_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
- public static final ByteSizeValue INACTIVE_SHARD_INDEXING_BUFFER = ByteSizeValue.parseBytesSizeValue("500kb", "INACTIVE_SHARD_INDEXING_BUFFER");
public static final String DEFAULT_VERSION_MAP_SIZE = "25%";
@@ -139,7 +138,8 @@ public final class EngineConfig {
this.failedEngineListener = failedEngineListener;
this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME);
- indexingBufferSize = DEFAULT_INDEX_BUFFER_SIZE;
+ // We start up inactive and rely on IndexingMemoryController to give us our fair share once we start indexing:
+ indexingBufferSize = IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER;
gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis();
versionMapSizeSetting = indexSettings.get(INDEX_VERSION_MAP_SIZE, DEFAULT_VERSION_MAP_SIZE);
updateVersionMapSize();
@@ -258,10 +258,10 @@ public final class EngineConfig {
/**
* Returns a {@link org.elasticsearch.index.indexing.ShardIndexingService} used inside the engine to inform about
- * pre and post index and create operations. The operations are used for statistic purposes etc.
+ * pre and post index. The operations are used for statistic purposes etc.
*
- * @see org.elasticsearch.index.indexing.ShardIndexingService#postCreate(org.elasticsearch.index.engine.Engine.Create)
- * @see org.elasticsearch.index.indexing.ShardIndexingService#preCreate(org.elasticsearch.index.engine.Engine.Create)
+ * @see org.elasticsearch.index.indexing.ShardIndexingService#postIndex(Engine.Index)
+ * @see org.elasticsearch.index.indexing.ShardIndexingService#preIndex(Engine.Index)
*
*/
public ShardIndexingService getIndexingService() {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineException.java b/core/src/main/java/org/elasticsearch/index/engine/EngineException.java
index d7487ef66f1..23f6be7ffd2 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineException.java
@@ -30,16 +30,16 @@ import java.io.IOException;
*/
public class EngineException extends ElasticsearchException {
- public EngineException(ShardId shardId, String msg) {
- this(shardId, msg, null);
+ public EngineException(ShardId shardId, String msg, Object... params) {
+ this(shardId, msg, null, params);
}
- public EngineException(ShardId shardId, String msg, Throwable cause) {
- super(msg, cause);
+ public EngineException(ShardId shardId, String msg, Throwable cause, Object... params) {
+ super(msg, cause, params);
setShard(shardId);
}
- public EngineException(StreamInput in) throws IOException{
+ public EngineException(StreamInput in) throws IOException {
super(in);
}
}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 227212dd86e..3973b47f3ac 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -21,8 +21,9 @@ package org.elasticsearch.index.engine;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@@ -31,7 +32,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.routing.DjbHashFunction;
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.logging.ESLogger;
@@ -48,7 +49,6 @@ import org.elasticsearch.index.indexing.ShardIndexingService;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.merge.OnGoingMerge;
-import org.elasticsearch.index.search.nested.IncludeNestedDocsQuery;
import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
import org.elasticsearch.index.shard.MergeSchedulerConfig;
import org.elasticsearch.index.shard.ShardId;
@@ -67,7 +67,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
-import java.util.function.Supplier;
/**
*
@@ -182,8 +181,7 @@ public class InternalEngine extends Engine {
}
translogConfig.setTranslogGeneration(generation);
if (generation != null && generation.translogUUID == null) {
- // only upgrade on pre-2.0 indices...
- Translog.upgradeLegacyTranslog(logger, translogConfig);
+ throw new IndexFormatTooOldException("trasnlog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
}
}
final Translog translog = new Translog(translogConfig);
@@ -316,7 +314,8 @@ public class InternalEngine extends Engine {
}
if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) {
Uid uid = Uid.createUid(get.uid().text());
- throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), versionValue.version(), get.version());
+ throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
+ get.versionType().explainConflictForReads(versionValue.version(), get.version()));
}
Translog.Operation op = translog.read(versionValue.translogLocation());
if (op != null) {
@@ -331,96 +330,7 @@ public class InternalEngine extends Engine {
}
@Override
- public void create(Create create) throws EngineException {
- try (ReleasableLock lock = readLock.acquire()) {
- ensureOpen();
- if (create.origin() == Operation.Origin.RECOVERY) {
- // Don't throttle recovery operations
- innerCreate(create);
- } else {
- try (Releasable r = throttle.acquireThrottle()) {
- innerCreate(create);
- }
- }
- } catch (OutOfMemoryError | IllegalStateException | IOException t) {
- maybeFailEngine("create", t);
- throw new CreateFailedEngineException(shardId, create.type(), create.id(), t);
- }
- checkVersionMapRefresh();
- }
-
- private void innerCreate(Create create) throws IOException {
- synchronized (dirtyLock(create.uid())) {
- final long currentVersion;
- final VersionValue versionValue;
- versionValue = versionMap.getUnderLock(create.uid().bytes());
- if (versionValue == null) {
- currentVersion = loadCurrentVersionFromIndex(create.uid());
- } else {
- if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > engineConfig.getGcDeletesInMillis()) {
- currentVersion = Versions.NOT_FOUND; // deleted, and GC
- } else {
- currentVersion = versionValue.version();
- }
- }
- innerCreateUnderLock(create, currentVersion, versionValue);
- }
- }
-
- private void innerCreateUnderLock(Create create, long currentVersion, VersionValue versionValue) throws IOException {
-
- // same logic as index
- long updatedVersion;
- long expectedVersion = create.version();
- if (create.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
- if (create.origin() == Operation.Origin.RECOVERY) {
- return;
- } else {
- throw new VersionConflictEngineException(shardId, create.type(), create.id(), currentVersion, expectedVersion);
- }
- }
- updatedVersion = create.versionType().updateVersion(currentVersion, expectedVersion);
-
- // if the doc exists
- boolean doUpdate = false;
- if ((versionValue != null && versionValue.delete() == false) || (versionValue == null && currentVersion != Versions.NOT_FOUND)) {
- if (create.origin() == Operation.Origin.RECOVERY) {
- return;
- } else if (create.origin() == Operation.Origin.REPLICA) {
- // #7142: the primary already determined it's OK to index this document, and we confirmed above that the version doesn't
- // conflict, so we must also update here on the replica to remain consistent:
- doUpdate = true;
- } else {
- // On primary, we throw DAEE if the _uid is already in the index with an older version:
- assert create.origin() == Operation.Origin.PRIMARY;
- throw new DocumentAlreadyExistsException(shardId, create.type(), create.id());
- }
- }
-
- create.updateVersion(updatedVersion);
-
- if (doUpdate) {
- if (create.docs().size() > 1) {
- indexWriter.updateDocuments(create.uid(), create.docs());
- } else {
- indexWriter.updateDocument(create.uid(), create.docs().get(0));
- }
- } else {
- if (create.docs().size() > 1) {
- indexWriter.addDocuments(create.docs());
- } else {
- indexWriter.addDocument(create.docs().get(0));
- }
- }
- Translog.Location translogLocation = translog.add(new Translog.Create(create));
-
- versionMap.putUnderLock(create.uid().bytes(), new VersionValue(updatedVersion, translogLocation));
- create.setTranslogLocation(translogLocation);
- indexingService.postCreateUnderLock(create);
- }
-
- @Override
- public boolean index(Index index) throws EngineException {
+ public boolean index(Index index) {
final boolean created;
try (ReleasableLock lock = readLock.acquire()) {
ensureOpen();
@@ -440,6 +350,67 @@ public class InternalEngine extends Engine {
return created;
}
+ private boolean innerIndex(Index index) throws IOException {
+ synchronized (dirtyLock(index.uid())) {
+ final long currentVersion;
+ final boolean deleted;
+ VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes());
+ if (versionValue == null) {
+ currentVersion = loadCurrentVersionFromIndex(index.uid());
+ deleted = currentVersion == Versions.NOT_FOUND;
+ } else {
+ deleted = versionValue.delete();
+ if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > engineConfig.getGcDeletesInMillis()) {
+ currentVersion = Versions.NOT_FOUND; // deleted, and GC
+ } else {
+ currentVersion = versionValue.version();
+ }
+ }
+
+ long expectedVersion = index.version();
+ if (index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
+ if (index.origin() == Operation.Origin.RECOVERY) {
+ return false;
+ } else {
+ throw new VersionConflictEngineException(shardId, index.type(), index.id(),
+ index.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
+ }
+ }
+ long updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion);
+
+ final boolean created;
+ index.updateVersion(updatedVersion);
+
+ if (currentVersion == Versions.NOT_FOUND) {
+ // document does not exists, we can optimize for create
+ created = true;
+ if (index.docs().size() > 1) {
+ indexWriter.addDocuments(index.docs());
+ } else {
+ indexWriter.addDocument(index.docs().get(0));
+ }
+ } else {
+ if (versionValue != null) {
+ created = versionValue.delete(); // we have a delete which is not GC'ed...
+ } else {
+ created = false;
+ }
+ if (index.docs().size() > 1) {
+ indexWriter.updateDocuments(index.uid(), index.docs());
+ } else {
+ indexWriter.updateDocument(index.uid(), index.docs().get(0));
+ }
+ }
+ Translog.Location translogLocation = translog.add(new Translog.Index(index));
+
+ versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, translogLocation));
+ index.setTranslogLocation(translogLocation);
+
+ indexingService.postIndexUnderLock(index);
+ return created;
+ }
+ }
+
/**
* Forces a refresh if the versionMap is using too much RAM
*/
@@ -467,62 +438,6 @@ public class InternalEngine extends Engine {
}
}
- private boolean innerIndex(Index index) throws IOException {
- synchronized (dirtyLock(index.uid())) {
- final long currentVersion;
- VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes());
- if (versionValue == null) {
- currentVersion = loadCurrentVersionFromIndex(index.uid());
- } else {
- if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > engineConfig.getGcDeletesInMillis()) {
- currentVersion = Versions.NOT_FOUND; // deleted, and GC
- } else {
- currentVersion = versionValue.version();
- }
- }
-
- long updatedVersion;
- long expectedVersion = index.version();
- if (index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
- if (index.origin() == Operation.Origin.RECOVERY) {
- return false;
- } else {
- throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion);
- }
- }
- updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion);
-
- final boolean created;
- index.updateVersion(updatedVersion);
- if (currentVersion == Versions.NOT_FOUND) {
- // document does not exists, we can optimize for create
- created = true;
- if (index.docs().size() > 1) {
- indexWriter.addDocuments(index.docs());
- } else {
- indexWriter.addDocument(index.docs().get(0));
- }
- } else {
- if (versionValue != null) {
- created = versionValue.delete(); // we have a delete which is not GC'ed...
- } else {
- created = false;
- }
- if (index.docs().size() > 1) {
- indexWriter.updateDocuments(index.uid(), index.docs());
- } else {
- indexWriter.updateDocument(index.uid(), index.docs().get(0));
- }
- }
- Translog.Location translogLocation = translog.add(new Translog.Index(index));
-
- versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, translogLocation));
- index.setTranslogLocation(translogLocation);
- indexingService.postIndexUnderLock(index);
- return created;
- }
- }
-
@Override
public void delete(Delete delete) throws EngineException {
try (ReleasableLock lock = readLock.acquire()) {
@@ -549,10 +464,13 @@ public class InternalEngine extends Engine {
private void innerDelete(Delete delete) throws IOException {
synchronized (dirtyLock(delete.uid())) {
final long currentVersion;
+ final boolean deleted;
VersionValue versionValue = versionMap.getUnderLock(delete.uid().bytes());
if (versionValue == null) {
currentVersion = loadCurrentVersionFromIndex(delete.uid());
+ deleted = currentVersion == Versions.NOT_FOUND;
} else {
+ deleted = versionValue.delete();
if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > engineConfig.getGcDeletesInMillis()) {
currentVersion = Versions.NOT_FOUND; // deleted, and GC
} else {
@@ -562,11 +480,12 @@ public class InternalEngine extends Engine {
long updatedVersion;
long expectedVersion = delete.version();
- if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) {
+ if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
if (delete.origin() == Operation.Origin.RECOVERY) {
return;
} else {
- throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, expectedVersion);
+ throw new VersionConflictEngineException(shardId, delete.type(), delete.id(),
+ delete.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
}
}
updatedVersion = delete.versionType().updateVersion(currentVersion, expectedVersion);
@@ -591,48 +510,6 @@ public class InternalEngine extends Engine {
}
}
- /** @deprecated This was removed, but we keep this API so translog can replay any DBQs on upgrade. */
- @Deprecated
- @Override
- public void delete(DeleteByQuery delete) throws EngineException {
- try (ReleasableLock lock = readLock.acquire()) {
- ensureOpen();
- if (delete.origin() == Operation.Origin.RECOVERY) {
- // Don't throttle recovery operations
- innerDelete(delete);
- } else {
- try (Releasable r = throttle.acquireThrottle()) {
- innerDelete(delete);
- }
- }
- }
- }
-
- private void innerDelete(DeleteByQuery delete) throws EngineException {
- try {
- Query query = delete.query();
- if (delete.aliasFilter() != null) {
- query = new BooleanQuery.Builder()
- .add(query, Occur.MUST)
- .add(delete.aliasFilter(), Occur.FILTER)
- .build();
- }
- if (delete.nested()) {
- query = new IncludeNestedDocsQuery(query, delete.parentFilter());
- }
-
- indexWriter.deleteDocuments(query);
- translog.add(new Translog.DeleteByQuery(delete));
- } catch (Throwable t) {
- maybeFailEngine("delete_by_query", t);
- throw new DeleteByQueryFailedEngineException(shardId, delete, t);
- }
-
- // TODO: This is heavy, since we refresh, but we must do this because we don't know which documents were in fact deleted (i.e., our
- // versionMap isn't updated), so we must force a cutover to a new reader to "see" the deletions:
- refresh("delete_by_query");
- }
-
@Override
public void refresh(String source) throws EngineException {
// we obtain a read lock here, since we don't want a flush to happen while we are refreshing
@@ -904,6 +781,11 @@ public class InternalEngine extends Engine {
stats.addIndexWriterMaxMemoryInBytes((long) (indexWriter.getConfig().getRAMBufferSizeMB() * 1024 * 1024));
}
+ @Override
+ public long indexWriterRAMBytesUsed() {
+ return indexWriter.ramBytesUsed();
+ }
+
@Override
public List segments(boolean verbose) {
try (ReleasableLock lock = readLock.acquire()) {
@@ -974,7 +856,7 @@ public class InternalEngine extends Engine {
}
private Object dirtyLock(BytesRef uid) {
- int hash = DjbHashFunction.DJB_HASH(uid.bytes, uid.offset, uid.length);
+ int hash = Murmur3HashFunction.hash(uid.bytes, uid.offset, uid.length);
return dirtyLocks[MathUtils.mod(hash, dirtyLocks.length)];
}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Segment.java b/core/src/main/java/org/elasticsearch/index/engine/Segment.java
index cbccaa1959a..7d3882fd9b6 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Segment.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Segment.java
@@ -19,7 +19,6 @@
package org.elasticsearch.index.engine;
-import com.google.common.collect.Iterators;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
import org.elasticsearch.common.Nullable;
@@ -32,7 +31,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
public class Segment implements Streamable {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index 7588ffae355..921f1167f43 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -102,11 +102,6 @@ public class ShadowEngine extends Engine {
}
- @Override
- public void create(Create create) throws EngineException {
- throw new UnsupportedOperationException(shardId + " create operation not allowed on shadow engine");
- }
-
@Override
public boolean index(Index index) throws EngineException {
throw new UnsupportedOperationException(shardId + " index operation not allowed on shadow engine");
@@ -117,13 +112,6 @@ public class ShadowEngine extends Engine {
throw new UnsupportedOperationException(shardId + " delete operation not allowed on shadow engine");
}
- /** @deprecated This was removed, but we keep this API so translog can replay any DBQs on upgrade. */
- @Deprecated
- @Override
- public void delete(DeleteByQuery delete) throws EngineException {
- throw new UnsupportedOperationException(shardId + " delete-by-query operation not allowed on shadow engine");
- }
-
@Override
public SyncedFlushResult syncFlush(String syncId, CommitId expectedCommitId) {
throw new UnsupportedOperationException(shardId + " sync commit operation not allowed on shadow engine");
@@ -245,4 +233,9 @@ public class ShadowEngine extends Engine {
return lastCommittedSegmentInfos;
}
+ @Override
+ public long indexWriterRAMBytesUsed() {
+ // No IndexWriter
+ throw new UnsupportedOperationException("ShadowEngine has no IndexWriter");
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java b/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
index 8c2d35297e7..9b038c6e77c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionConflictEngineException.java
@@ -29,8 +29,16 @@ import java.io.IOException;
*/
public class VersionConflictEngineException extends EngineException {
- public VersionConflictEngineException(ShardId shardId, String type, String id, long current, long provided) {
- super(shardId, "[" + type + "][" + id + "]: version conflict, current [" + current + "], provided [" + provided + "]");
+ public VersionConflictEngineException(ShardId shardId, String type, String id, String explanation) {
+ this(shardId, null, type, id, explanation);
+ }
+
+ public VersionConflictEngineException(ShardId shardId, Throwable cause, String type, String id, String explanation) {
+ this(shardId, "[{}][{}]: version conflict, {}", cause, type, id, explanation);
+ }
+
+ public VersionConflictEngineException(ShardId shardId, String msg, Throwable cause, Object... params) {
+ super(shardId, msg, cause, params);
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/index/indexing/IndexingOperationListener.java b/core/src/main/java/org/elasticsearch/index/indexing/IndexingOperationListener.java
index 858453fcba4..651bc405a84 100644
--- a/core/src/main/java/org/elasticsearch/index/indexing/IndexingOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/indexing/IndexingOperationListener.java
@@ -28,39 +28,8 @@ public abstract class IndexingOperationListener {
/**
* Called before the indexing occurs.
*/
- public Engine.Create preCreate(Engine.Create create) {
- return create;
- }
-
- /**
- * Called after the indexing occurs, under a locking scheme to maintain
- * concurrent updates to the same doc.
- *
- * Note, long operations should not occur under this callback.
- */
- public void postCreateUnderLock(Engine.Create create) {
-
- }
-
- /**
- * Called after create index operation occurred.
- */
- public void postCreate(Engine.Create create) {
-
- }
-
- /**
- * Called after create index operation occurred with exception.
- */
- public void postCreate(Engine.Create create, Throwable ex) {
-
- }
-
- /**
- * Called before the indexing occurs.
- */
- public Engine.Index preIndex(Engine.Index index) {
- return index;
+ public Engine.Index preIndex(Engine.Index operation) {
+ return operation;
}
/**
diff --git a/core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java
index ea45db2e912..292c2a16e91 100644
--- a/core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java
@@ -128,10 +128,6 @@ public final class IndexingSlowLog {
postIndexing(index.parsedDoc(), tookInNanos);
}
- void postCreate(Engine.Create create, long tookInNanos) {
- postIndexing(create.parsedDoc(), tookInNanos);
- }
-
/**
* Reads how much of the source to log. The user can specify any value they
* like and numbers are interpreted the maximum number of characters to log
diff --git a/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java b/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java
index b61ba7fe04d..a3a1fa5b4a7 100644
--- a/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java
+++ b/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java
@@ -86,25 +86,6 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
listeners.remove(listener);
}
- public Engine.Create preCreate(Engine.Create create) {
- totalStats.indexCurrent.inc();
- typeStats(create.type()).indexCurrent.inc();
- for (IndexingOperationListener listener : listeners) {
- create = listener.preCreate(create);
- }
- return create;
- }
-
- public void postCreateUnderLock(Engine.Create create) {
- for (IndexingOperationListener listener : listeners) {
- try {
- listener.postCreateUnderLock(create);
- } catch (Exception e) {
- logger.warn("postCreateUnderLock listener [{}] failed", e, listener);
- }
- }
- }
-
public void throttlingActivated() {
totalStats.setThrottled(true);
}
@@ -113,40 +94,13 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
totalStats.setThrottled(false);
}
- public void postCreate(Engine.Create create) {
- long took = create.endTime() - create.startTime();
- totalStats.indexMetric.inc(took);
- totalStats.indexCurrent.dec();
- StatsHolder typeStats = typeStats(create.type());
- typeStats.indexMetric.inc(took);
- typeStats.indexCurrent.dec();
- slowLog.postCreate(create, took);
- for (IndexingOperationListener listener : listeners) {
- try {
- listener.postCreate(create);
- } catch (Exception e) {
- logger.warn("postCreate listener [{}] failed", e, listener);
- }
- }
- }
-
- public void postCreate(Engine.Create create, Throwable ex) {
- for (IndexingOperationListener listener : listeners) {
- try {
- listener.postCreate(create, ex);
- } catch (Throwable t) {
- logger.warn("postCreate listener [{}] failed", t, listener);
- }
- }
- }
-
- public Engine.Index preIndex(Engine.Index index) {
+ public Engine.Index preIndex(Engine.Index operation) {
totalStats.indexCurrent.inc();
- typeStats(index.type()).indexCurrent.inc();
+ typeStats(operation.type()).indexCurrent.inc();
for (IndexingOperationListener listener : listeners) {
- index = listener.preIndex(index);
+ operation = listener.preIndex(operation);
}
- return index;
+ return operation;
}
public void postIndexUnderLock(Engine.Index index) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
index 65a6d3ae91d..91371a2663a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
@@ -64,7 +64,7 @@ import org.elasticsearch.index.mapper.ip.IpFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
+import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
@@ -86,7 +86,7 @@ public class DocumentMapperParser {
final MapperService mapperService;
final AnalysisService analysisService;
private static final ESLogger logger = Loggers.getLogger(DocumentMapperParser.class);
- private final SimilarityLookupService similarityLookupService;
+ private final SimilarityService similarityService;
private final ScriptService scriptService;
private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
@@ -100,12 +100,12 @@ public class DocumentMapperParser {
private volatile SortedMap additionalRootMappers;
public DocumentMapperParser(@IndexSettings Settings indexSettings, MapperService mapperService, AnalysisService analysisService,
- SimilarityLookupService similarityLookupService, ScriptService scriptService) {
+ SimilarityService similarityService, ScriptService scriptService) {
this.indexSettings = indexSettings;
this.parseFieldMatcher = new ParseFieldMatcher(indexSettings);
this.mapperService = mapperService;
this.analysisService = analysisService;
- this.similarityLookupService = similarityLookupService;
+ this.similarityService = similarityService;
this.scriptService = scriptService;
Map typeParsers = new HashMap<>();
typeParsers.put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser());
@@ -170,7 +170,7 @@ public class DocumentMapperParser {
}
public Mapper.TypeParser.ParserContext parserContext(String type) {
- return new Mapper.TypeParser.ParserContext(type, analysisService, similarityLookupService, mapperService, typeParsers, indexVersionCreated, parseFieldMatcher);
+ return new Mapper.TypeParser.ParserContext(type, analysisService, similarityService::getSimilarity, mapperService, typeParsers::get, indexVersionCreated, parseFieldMatcher);
}
public DocumentMapper parse(String source) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index db2919e0217..97435e039e1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -122,7 +122,7 @@ class DocumentParser implements Closeable {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
- Mapper update = parseObject(context, mapping.root);
+ Mapper update = parseObject(context, mapping.root, true);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
@@ -194,7 +194,7 @@ class DocumentParser implements Closeable {
return doc;
}
- static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper) throws IOException {
+ static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
if (mapper.isEnabled() == false) {
context.parser().skipChildren();
return null;
@@ -202,6 +202,10 @@ class DocumentParser implements Closeable {
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
+ if (atRoot && MapperService.isMetadataField(currentFieldName) &&
+ Version.indexCreated(context.indexSettings()).onOrAfter(Version.V_2_0_0_beta1)) {
+ throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside a document. Use the index API request parameters.");
+ }
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
@@ -302,7 +306,7 @@ class DocumentParser implements Closeable {
private static Mapper parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
if (mapper instanceof ObjectMapper) {
- return parseObject(context, (ObjectMapper) mapper);
+ return parseObject(context, (ObjectMapper) mapper, false);
} else {
FieldMapper fieldMapper = (FieldMapper)mapper;
Mapper update = fieldMapper.parse(context);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index ec53fbaa0b4..45bef68ee00 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -34,8 +34,8 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.core.TypeParsers;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
import org.elasticsearch.index.similarity.SimilarityProvider;
+import org.elasticsearch.index.similarity.SimilarityService;
import java.io.IOException;
import java.util.ArrayList;
@@ -447,7 +447,7 @@ public abstract class FieldMapper extends Mapper {
if (fieldType().similarity() != null) {
builder.field("similarity", fieldType().similarity().name());
} else if (includeDefaults) {
- builder.field("similarity", SimilarityLookupService.DEFAULT_SIMILARITY);
+ builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
if (includeDefaults || hasCustomFieldDataSettings()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index f55ca93c276..9ca34e1c573 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -26,9 +26,10 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
+import org.elasticsearch.index.similarity.SimilarityProvider;
import java.util.Map;
+import java.util.function.Function;
public abstract class Mapper implements ToXContent, Iterable {
@@ -84,18 +85,18 @@ public abstract class Mapper implements ToXContent, Iterable {
private final AnalysisService analysisService;
- private final SimilarityLookupService similarityLookupService;
+ private final Function similarityLookupService;
private final MapperService mapperService;
- private final Map typeParsers;
+ private final Function typeParsers;
private final Version indexVersionCreated;
private final ParseFieldMatcher parseFieldMatcher;
- public ParserContext(String type, AnalysisService analysisService, SimilarityLookupService similarityLookupService,
- MapperService mapperService, Map typeParsers,
+ public ParserContext(String type, AnalysisService analysisService, Function similarityLookupService,
+ MapperService mapperService, Function typeParsers,
Version indexVersionCreated, ParseFieldMatcher parseFieldMatcher) {
this.type = type;
this.analysisService = analysisService;
@@ -114,8 +115,8 @@ public abstract class Mapper implements ToXContent, Iterable {
return analysisService;
}
- public SimilarityLookupService similarityLookupService() {
- return similarityLookupService;
+ public SimilarityProvider getSimilarity(String name) {
+ return similarityLookupService.apply(name);
}
public MapperService mapperService() {
@@ -123,7 +124,7 @@ public abstract class Mapper implements ToXContent, Iterable {
}
public TypeParser typeParser(String type) {
- return typeParsers.get(Strings.toUnderscoreCase(type));
+ return typeParsers.apply(Strings.toUnderscoreCase(type));
}
public Version indexVersionCreated() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 0357ef24060..bbd96f7d930 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper;
import com.carrotsearch.hppc.ObjectHashSet;
-import com.google.common.collect.Iterators;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
@@ -51,7 +50,7 @@ import org.elasticsearch.index.mapper.Mapper.BuilderContext;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
+import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.InvalidTypeNameException;
import org.elasticsearch.indices.TypeMissingException;
import org.elasticsearch.percolator.PercolatorService;
@@ -65,13 +64,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
+import java.util.stream.Collectors;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
@@ -126,12 +125,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
@Inject
public MapperService(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService,
- SimilarityLookupService similarityLookupService,
+ SimilarityService similarityService,
ScriptService scriptService) {
super(index, indexSettings);
this.analysisService = analysisService;
this.fieldTypes = new FieldTypeLookup();
- this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityLookupService, scriptService);
+ this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService, scriptService);
this.indexAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), p -> p.indexAnalyzer());
this.searchAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchAnalyzer(), p -> p.searchAnalyzer());
this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
@@ -186,13 +185,13 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
*/
public Iterable docMappers(final boolean includingDefaultMapping) {
return () -> {
- final Iterator iterator;
+ final Collection documentMappers;
if (includingDefaultMapping) {
- iterator = mappers.values().iterator();
+ documentMappers = mappers.values();
} else {
- iterator = mappers.values().stream().filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type())).iterator();
+ documentMappers = mappers.values().stream().filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type())).collect(Collectors.toList());
}
- return Iterators.unmodifiableIterator(iterator);
+ return Collections.unmodifiableCollection(documentMappers).iterator();
};
}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index 78d038526b3..7468f4fb2f6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -79,7 +79,6 @@ public class BinaryFieldMapper extends FieldMapper {
@Override
public BinaryFieldMapper build(BuilderContext context) {
setupFieldType(context);
- ((BinaryFieldType)fieldType).setTryUncompressing(context.indexCreatedVersion().before(Version.V_2_0_0_beta1));
return new BinaryFieldMapper(name, fieldType, defaultFieldType,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
@@ -103,13 +102,11 @@ public class BinaryFieldMapper extends FieldMapper {
}
static final class BinaryFieldType extends MappedFieldType {
- private boolean tryUncompressing = false;
public BinaryFieldType() {}
protected BinaryFieldType(BinaryFieldType ref) {
super(ref);
- this.tryUncompressing = ref.tryUncompressing;
}
@Override
@@ -117,40 +114,12 @@ public class BinaryFieldMapper extends FieldMapper {
return new BinaryFieldType(this);
}
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o)) return false;
- BinaryFieldType that = (BinaryFieldType) o;
- return Objects.equals(tryUncompressing, that.tryUncompressing);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), tryUncompressing);
- }
@Override
public String typeName() {
return CONTENT_TYPE;
}
- @Override
- public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) {
- super.checkCompatibility(fieldType, conflicts, strict);
- BinaryFieldType other = (BinaryFieldType)fieldType;
- if (tryUncompressing() != other.tryUncompressing()) {
- conflicts.add("mapper [" + names().fullName() + "] has different [try_uncompressing] (IMPOSSIBLE)");
- }
- }
-
- public boolean tryUncompressing() {
- return tryUncompressing;
- }
-
- public void setTryUncompressing(boolean tryUncompressing) {
- checkIfFrozen();
- this.tryUncompressing = tryUncompressing;
- }
@Override
public BytesReference value(Object value) {
@@ -172,15 +141,7 @@ public class BinaryFieldMapper extends FieldMapper {
throw new ElasticsearchParseException("failed to convert bytes", e);
}
}
- try {
- if (tryUncompressing) { // backcompat behavior
- return CompressorFactory.uncompressIfNeeded(bytes);
- } else {
- return bytes;
- }
- } catch (IOException e) {
- throw new ElasticsearchParseException("failed to decompress source", e);
- }
+ return bytes;
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
index 7f06c223e62..0e512bf4281 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
@@ -19,8 +19,6 @@
package org.elasticsearch.index.mapper.core;
-import com.carrotsearch.hppc.DoubleArrayList;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
@@ -36,8 +34,6 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.util.ByteUtils;
-import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -286,17 +282,7 @@ public class DoubleFieldMapper extends NumberFieldMapper {
fields.add(field);
}
if (fieldType().hasDocValues()) {
- if (useSortedNumericDocValues) {
- addDocValue(context, fields, doubleToSortableLong(value));
- } else {
- CustomDoubleNumericDocValuesField field = (CustomDoubleNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
- if (field != null) {
- field.add(value);
- } else {
- field = new CustomDoubleNumericDocValuesField(fieldType().names().indexName(), value);
- context.doc().addWithKey(fieldType().names().indexName(), field);
- }
- }
+ addDocValue(context, fields, doubleToSortableLong(value));
}
}
@@ -346,30 +332,4 @@ public class DoubleFieldMapper extends NumberFieldMapper {
}
}
- public static class CustomDoubleNumericDocValuesField extends CustomNumericDocValuesField {
-
- private final DoubleArrayList values;
-
- public CustomDoubleNumericDocValuesField(String name, double value) {
- super(name);
- values = new DoubleArrayList();
- add(value);
- }
-
- public void add(double value) {
- values.add(value);
- }
-
- @Override
- public BytesRef binaryValue() {
- CollectionUtils.sortAndDedup(values);
-
- final byte[] bytes = new byte[values.size() * 8];
- for (int i = 0; i < values.size(); ++i) {
- ByteUtils.writeDoubleLE(values.get(i), bytes, i * 8);
- }
- return new BytesRef(bytes);
- }
-
- }
}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index caeb2d7a188..9a607ffd415 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -19,8 +19,6 @@
package org.elasticsearch.index.mapper.core;
-import com.carrotsearch.hppc.FloatArrayList;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
@@ -37,8 +35,6 @@ import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.util.ByteUtils;
-import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -298,17 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper {
fields.add(field);
}
if (fieldType().hasDocValues()) {
- if (useSortedNumericDocValues) {
- addDocValue(context, fields, floatToSortableInt(value));
- } else {
- CustomFloatNumericDocValuesField field = (CustomFloatNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
- if (field != null) {
- field.add(value);
- } else {
- field = new CustomFloatNumericDocValuesField(fieldType().names().indexName(), value);
- context.doc().addWithKey(fieldType().names().indexName(), field);
- }
- }
+ addDocValue(context, fields, floatToSortableInt(value));
}
}
@@ -357,31 +343,4 @@ public class FloatFieldMapper extends NumberFieldMapper {
return Float.toString(number);
}
}
-
- public static class CustomFloatNumericDocValuesField extends CustomNumericDocValuesField {
-
- private final FloatArrayList values;
-
- public CustomFloatNumericDocValuesField(String name, float value) {
- super(name);
- values = new FloatArrayList();
- add(value);
- }
-
- public void add(float value) {
- values.add(value);
- }
-
- @Override
- public BytesRef binaryValue() {
- CollectionUtils.sortAndDedup(values);
-
- final byte[] bytes = new byte[values.size() * 4];
- for (int i = 0; i < values.size(); ++i) {
- ByteUtils.writeFloatLE(values.get(i), bytes, i * 4);
- }
- return new BytesRef(bytes);
- }
-
- }
}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 78406c2afbc..3fba511fb52 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper.core;
-import com.carrotsearch.hppc.LongArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
@@ -31,14 +30,10 @@ import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.search.Query;
-import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.util.ByteUtils;
-import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.*;
@@ -170,21 +165,12 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
protected Explicit coerce;
- /**
- * True if index version is 1.4+
- *
- * In this case numerics are encoded with SORTED_NUMERIC docvalues,
- * otherwise for older indexes we must continue to write BINARY (for now)
- */
- protected final boolean useSortedNumericDocValues;
-
protected NumberFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit ignoreMalformed, Explicit coerce, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.ignoreMalformed = ignoreMalformed;
this.coerce = coerce;
- this.useSortedNumericDocValues = Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1);
}
@Override
@@ -225,17 +211,7 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
protected abstract void innerParseCreateField(ParseContext context, List fields) throws IOException;
protected final void addDocValue(ParseContext context, List fields, long value) {
- if (useSortedNumericDocValues) {
- fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value));
- } else {
- CustomLongNumericDocValuesField field = (CustomLongNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
- if (field != null) {
- field.add(value);
- } else {
- field = new CustomLongNumericDocValuesField(fieldType().names().indexName(), value);
- context.doc().addWithKey(fieldType().names().indexName(), field);
- }
- }
+ fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value));
}
/**
@@ -414,40 +390,6 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
}
-
- public static class CustomLongNumericDocValuesField extends CustomNumericDocValuesField {
-
- private final LongArrayList values;
-
- public CustomLongNumericDocValuesField(String name, long value) {
- super(name);
- values = new LongArrayList();
- add(value);
- }
-
- public void add(long value) {
- values.add(value);
- }
-
- @Override
- public BytesRef binaryValue() {
- CollectionUtils.sortAndDedup(values);
-
- // here is the trick:
- // - the first value is zig-zag encoded so that eg. -5 would become positive and would be better compressed by vLong
- // - for other values, we only encode deltas using vLong
- final byte[] bytes = new byte[values.size() * ByteUtils.MAX_BYTES_VLONG];
- final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
- ByteUtils.writeVLong(out, ByteUtils.zigZagEncode(values.get(0)));
- for (int i = 1; i < values.size(); ++i) {
- final long delta = values.get(i) - values.get(i - 1);
- ByteUtils.writeVLong(out, delta);
- }
- return new BytesRef(bytes, 0, out.getPosition());
- }
-
- }
-
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
index 0588bd1e044..3f142cc2f9c 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
@@ -173,7 +173,7 @@ public class TypeParsers {
builder.omitNorms(nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("similarity")) {
- builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString()));
+ builder.similarity(parserContext.getSimilarity(propNode.toString()));
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
iterator.remove();
@@ -277,7 +277,7 @@ public class TypeParsers {
// ignore for old indexes
iterator.remove();
} else if (propName.equals("similarity")) {
- builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString()));
+ builder.similarity(parserContext.getSimilarity(propNode.toString()));
iterator.remove();
} else if (propName.equals("fielddata")) {
final Settings settings = Settings.builder().put(SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(propNode, "fielddata"))).build();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
index 4111786009e..b264bfa4bc3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.geo;
import com.carrotsearch.hppc.ObjectHashSet;
import com.carrotsearch.hppc.cursors.ObjectCursor;
-import com.google.common.collect.Iterators;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.util.BytesRef;
@@ -30,6 +29,7 @@ import org.apache.lucene.util.XGeoHashUtils;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
@@ -39,14 +39,7 @@ import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MergeMappingException;
-import org.elasticsearch.index.mapper.MergeResult;
-import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.DoubleFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper.CustomNumericDocValuesField;
@@ -54,18 +47,10 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
+import java.util.*;
-import static org.elasticsearch.index.mapper.MapperBuilders.doubleField;
-import static org.elasticsearch.index.mapper.MapperBuilders.geoPointField;
-import static org.elasticsearch.index.mapper.MapperBuilders.stringField;
-import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
-import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
-import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType;
+import static org.elasticsearch.index.mapper.MapperBuilders.*;
+import static org.elasticsearch.index.mapper.core.TypeParsers.*;
/**
* Parsing: We handle:
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index e538a00da16..59b664dbd65 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -41,7 +41,7 @@ import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.similarity.SimilarityLookupService;
+import org.elasticsearch.index.similarity.SimilarityService;
import java.io.IOException;
import java.util.Iterator;
@@ -300,7 +300,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
if (fieldType().similarity() != null) {
builder.field("similarity", fieldType().similarity().name());
} else if (includeDefaults) {
- builder.field("similarity", SimilarityLookupService.DEFAULT_SIMILARITY);
+ builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 1ac34df5063..1d73139fb25 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper.ip;
-import com.google.common.net.InetAddresses;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
@@ -29,6 +28,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
index d811f1f6e71..1f8a4c61f9a 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
@@ -242,29 +242,12 @@ public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent
private class RealTimePercolatorOperationListener extends IndexingOperationListener {
@Override
- public Engine.Create preCreate(Engine.Create create) {
+ public Engine.Index preIndex(Engine.Index operation) {
// validate the query here, before we index
- if (PercolatorService.TYPE_NAME.equals(create.type())) {
- parsePercolatorDocument(create.id(), create.source());
+ if (PercolatorService.TYPE_NAME.equals(operation.type())) {
+ parsePercolatorDocument(operation.id(), operation.source());
}
- return create;
- }
-
- @Override
- public void postCreateUnderLock(Engine.Create create) {
- // add the query under a doc lock
- if (PercolatorService.TYPE_NAME.equals(create.type())) {
- addPercolateQuery(create.id(), create.source());
- }
- }
-
- @Override
- public Engine.Index preIndex(Engine.Index index) {
- // validate the query here, before we index
- if (PercolatorService.TYPE_NAME.equals(index.type())) {
- parsePercolatorDocument(index.id(), index.source());
- }
- return index;
+ return operation;
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index 4ad63ebc1fc..59d20cef611 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -314,8 +314,8 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder shell = new ArrayList<>();
int size = in.readVInt();
for (int i = 0; i < size; i++) {
- shell.add(GeoPoint.readGeoPointFrom(in));
+ shell.add(in.readGeoPoint());
}
GeoPolygonQueryBuilder builder = new GeoPolygonQueryBuilder(fieldName, shell);
builder.validationMethod = GeoValidationMethod.readGeoValidationMethodFrom(in);
@@ -176,7 +176,7 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder {
- private static final DecayFunction EXP_DECAY_FUNCTION = new ExponentialDecayScoreFunction();
+ public static final DecayFunction EXP_DECAY_FUNCTION = new ExponentialDecayScoreFunction();
public ExponentialDecayFunctionBuilder(String fieldName, Object origin, Object scale, Object offset) {
super(fieldName, origin, scale, offset);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/gauss/GaussDecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/gauss/GaussDecayFunctionBuilder.java
index 621b22a583d..618503a9b3c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/gauss/GaussDecayFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/gauss/GaussDecayFunctionBuilder.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.query.functionscore.DecayFunctionBuilder;
public class GaussDecayFunctionBuilder extends DecayFunctionBuilder {
- private static final DecayFunction GAUSS_DECAY_FUNCTION = new GaussScoreFunction();
+ public static final DecayFunction GAUSS_DECAY_FUNCTION = new GaussScoreFunction();
public GaussDecayFunctionBuilder(String fieldName, Object origin, Object scale, Object offset) {
super(fieldName, origin, scale, offset);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/lin/LinearDecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/lin/LinearDecayFunctionBuilder.java
index 2e63aedd84b..f321ee166f1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/lin/LinearDecayFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/lin/LinearDecayFunctionBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.index.query.functionscore.DecayFunctionBuilder;
public class LinearDecayFunctionBuilder extends DecayFunctionBuilder {
- private static final DecayFunction LINEAR_DECAY_FUNCTION = new LinearDecayScoreFunction();
+ public static final DecayFunction LINEAR_DECAY_FUNCTION = new LinearDecayScoreFunction();
public LinearDecayFunctionBuilder(String fieldName, Object origin, Object scale, Object offset) {
super(fieldName, origin, scale, offset);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index ea2d555ae0d..86e53b41c9b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest;
import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
import org.elasticsearch.action.termvectors.TermVectorsRequest;
import org.elasticsearch.action.termvectors.TermVectorsResponse;
-import org.elasticsearch.bootstrap.Elasticsearch;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
@@ -43,6 +42,7 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.support.LoggerMessageFormat;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
@@ -84,8 +84,8 @@ import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.snapshots.IndexShardRepository;
-import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.Store.MetadataSnapshot;
+import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.index.suggest.stats.ShardSuggestMetric;
@@ -100,6 +100,7 @@ import org.elasticsearch.index.warmer.WarmerStats;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.InternalIndicesLifecycle;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
+import org.elasticsearch.indices.memory.IndexingMemoryController;
import org.elasticsearch.indices.recovery.RecoveryFailedException;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.percolator.PercolatorService;
@@ -118,16 +119,15 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+
public class IndexShard extends AbstractIndexShardComponent implements IndexSettingsService.Listener {
private final ThreadPool threadPool;
private final MapperService mapperService;
- private final IndexQueryParserService queryParserService;
private final IndexCache indexCache;
private final InternalIndicesLifecycle indicesLifecycle;
private final Store store;
private final MergeSchedulerConfig mergeSchedulerConfig;
- private final IndexAliasesService indexAliasesService;
private final ShardIndexingService indexingService;
private final ShardSearchStats searchService;
private final ShardGetService getService;
@@ -190,6 +190,13 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
private final IndexSearcherWrapper searcherWrapper;
+ /** True if this shard is still indexing (recently) and false if we've been idle for long enough (as periodically checked by {@link
+ * IndexingMemoryController}). */
+ private final AtomicBoolean active = new AtomicBoolean();
+
+ private volatile long lastWriteNS;
+ private final IndexingMemoryController indexingMemoryController;
+
@Inject
public IndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) {
super(shardId, indexSettings);
@@ -202,11 +209,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
this.indicesLifecycle = (InternalIndicesLifecycle) provider.getIndicesLifecycle();
this.store = store;
this.mergeSchedulerConfig = new MergeSchedulerConfig(indexSettings);
- this.threadPool = provider.getThreadPool();
- this.mapperService = provider.getMapperService();
- this.queryParserService = provider.getQueryParserService();
- this.indexCache = provider.getIndexCache();
- this.indexAliasesService = provider.getIndexAliasesService();
+ this.threadPool = provider.getThreadPool();
+ this.mapperService = provider.getMapperService();
+ this.indexCache = provider.getIndexCache();
this.indexingService = new ShardIndexingService(shardId, indexSettings);
this.getService = new ShardGetService(this, mapperService);
this.termVectorsService = provider.getTermVectorsService();
@@ -242,11 +247,16 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB));
this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false);
this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId);
+ this.indexingMemoryController = provider.getIndexingMemoryController();
+
this.searcherWrapper = provider.getIndexSearcherWrapper();
- this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryParserService, indexingService, mapperService, indexFieldDataService);
+ this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, provider.getQueryParserService(), indexingService, mapperService, indexFieldDataService);
if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) {
percolatorQueriesRegistry.enableRealTimePercolator();
}
+
+ // We start up inactive
+ active.set(false);
}
public Store store() {
@@ -278,7 +288,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
return indexFieldDataService;
}
- public MapperService mapperService() { return mapperService;}
+ public MapperService mapperService() {
+ return mapperService;
+ }
public ShardSearchStats searchService() {
return this.searchService;
@@ -423,40 +435,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
return previousState;
}
- public Engine.Create prepareCreate(SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin) {
- try {
- return prepareCreate(docMapper(source.type()), source, version, versionType, origin);
- } catch (Throwable t) {
- verifyNotClosed(t);
- throw t;
- }
- }
-
- static Engine.Create prepareCreate(DocumentMapperForType docMapper, SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin) {
- long startTime = System.nanoTime();
- ParsedDocument doc = docMapper.getDocumentMapper().parse(source);
- if (docMapper.getMapping() != null) {
- doc.addDynamicMappingsUpdate(docMapper.getMapping());
- }
- return new Engine.Create(docMapper.getDocumentMapper().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime);
- }
-
- public void create(Engine.Create create) {
- writeAllowed(create.origin());
- create = indexingService.preCreate(create);
- try {
- if (logger.isTraceEnabled()) {
- logger.trace("index [{}][{}]{}", create.type(), create.id(), create.docs());
- }
- getEngine().create(create);
- create.endTime(System.nanoTime());
- } catch (Throwable ex) {
- indexingService.postCreate(create, ex);
- throw ex;
- }
- indexingService.postCreate(create);
- }
-
public Engine.Index prepareIndex(SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin) {
try {
return prepareIndex(docMapper(source.type()), source, version, versionType, origin);
@@ -480,7 +458,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
* updated.
*/
public boolean index(Engine.Index index) {
- writeAllowed(index.origin());
+ ensureWriteAllowed(index);
+ markLastWrite(index);
index = indexingService.preIndex(index);
final boolean created;
try {
@@ -504,7 +483,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
}
public void delete(Engine.Delete delete) {
- writeAllowed(delete.origin());
+ ensureWriteAllowed(delete);
+ markLastWrite(delete);
delete = indexingService.preDelete(delete);
try {
if (logger.isTraceEnabled()) {
@@ -914,7 +894,24 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
}
}
- private void writeAllowed(Engine.Operation.Origin origin) throws IllegalIndexShardStateException {
+ /** Returns timestamp of last indexing operation */
+ public long getLastWriteNS() {
+ return lastWriteNS;
+ }
+
+ /** Records timestamp of the last write operation, possibly switching {@code active} to true if we were inactive. */
+ private void markLastWrite(Engine.Operation op) {
+ lastWriteNS = op.startTime();
+ if (active.getAndSet(true) == false) {
+ // We are currently inactive, but a new write operation just showed up, so we now notify IMC
+ // to wake up and fix our indexing buffer. We could do this async instead, but cost should
+ // be low, and it's rare this happens.
+ indexingMemoryController.forceCheck();
+ }
+ }
+
+ private void ensureWriteAllowed(Engine.Operation op) throws IllegalIndexShardStateException {
+ Engine.Operation.Origin origin = op.origin();
IndexShardState state = this.state; // one time volatile read
if (origin == Engine.Operation.Origin.PRIMARY) {
@@ -976,6 +973,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
this.failedEngineListener.delegates.add(failedEngineListener);
}
+ /** Change the indexing and translog buffer sizes. If {@code IndexWriter} is currently using more than
+ * the new buffering indexing size then we do a refresh to free up the heap. */
public void updateBufferSize(ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) {
final EngineConfig config = engineConfig;
@@ -994,27 +993,50 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
// so we push changes these changes down to IndexWriter:
engine.onSettingsChanged();
- if (shardIndexingBufferSize == EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER) {
- // it's inactive: make sure we do a refresh / full IW flush in this case, since the memory
- // changes only after a "data" change has happened to the writer
- // the index writer lazily allocates memory and a refresh will clean it all up.
- logger.debug("updating index_buffer_size from [{}] to (inactive) [{}]", preValue, shardIndexingBufferSize);
+ long iwBytesUsed = engine.indexWriterRAMBytesUsed();
+
+ String message = LoggerMessageFormat.format("updating index_buffer_size from [{}] to [{}]; IndexWriter now using [{}] bytes",
+ preValue, shardIndexingBufferSize, iwBytesUsed);
+
+ if (iwBytesUsed > shardIndexingBufferSize.bytes()) {
+ // our allowed buffer was changed to less than we are currently using; we ask IW to refresh
+ // so it clears its buffers (otherwise it won't clear until the next indexing/delete op)
+ logger.debug(message + "; now refresh to clear IndexWriter memory");
+
+ // TODO: should IW have an API to move segments to disk, but not refresh? Its flush method is protected...
try {
refresh("update index buffer");
} catch (Throwable e) {
- logger.warn("failed to refresh after setting shard to inactive", e);
+ logger.warn("failed to refresh after decreasing index buffer", e);
}
} else {
- logger.debug("updating index_buffer_size from [{}] to [{}]", preValue, shardIndexingBufferSize);
+ logger.debug(message);
}
}
engine.getTranslog().updateBuffer(shardTranslogBufferSize);
}
- public void markAsInactive() {
- updateBufferSize(EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER, TranslogConfig.INACTIVE_SHARD_TRANSLOG_BUFFER);
- indicesLifecycle.onShardInactive(this);
+ /** Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last
+ * indexing operation, and become inactive (reducing indexing and translog buffers to tiny values) if so. This returns true
+ * if the shard is inactive. */
+ public boolean checkIdle(long inactiveTimeNS) {
+ if (System.nanoTime() - lastWriteNS >= inactiveTimeNS) {
+ boolean wasActive = active.getAndSet(false);
+ if (wasActive) {
+ updateBufferSize(IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER, IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER);
+ logger.debug("shard is now inactive");
+ indicesLifecycle.onShardInactive(this);
+ }
+ }
+
+ return active.get() == false;
+ }
+
+ /** Returns {@code true} if this shard is active (has seen indexing ops in the last {@link
+ * IndexingMemoryController#SHARD_INACTIVE_TIME_SETTING} (default 5 minutes), else {@code false}. */
+ public boolean getActive() {
+ return active.get();
}
public final boolean isFlushOnClose() {
@@ -1416,8 +1438,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
}
private final EngineConfig newEngineConfig(TranslogConfig translogConfig, QueryCachingPolicy cachingPolicy) {
- final TranslogRecoveryPerformer translogRecoveryPerformer = new TranslogRecoveryPerformer(shardId, mapperService, queryParserService,
- indexAliasesService, indexCache, logger) {
+ final TranslogRecoveryPerformer translogRecoveryPerformer = new TranslogRecoveryPerformer(shardId, mapperService, logger) {
@Override
protected void operationProcessed() {
assert recoveryState != null;
@@ -1426,7 +1447,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
};
return new EngineConfig(shardId,
threadPool, indexingService, indexSettings, warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
- mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig);
+ mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig);
}
private static class IndexShardOperationCounter extends AbstractRefCounted {
@@ -1499,6 +1520,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett
/**
* Schedules a flush if needed but won't schedule more than one flush concurrently. The flush will be executed on the
* Flush thread-pool asynchronously.
+ *
* @return true if a new flush is scheduled otherwise false.
*/
public boolean maybeFlush() {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index c81b9e5c541..8bdf1fb5382 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -18,6 +18,8 @@
*/
package org.elasticsearch.index.shard;
+import java.io.IOException;
+
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexServicesProvider;
@@ -26,8 +28,7 @@ import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.Store;
-
-import java.io.IOException;
+import org.elasticsearch.index.translog.TranslogStats;
/**
* ShadowIndexShard extends {@link IndexShard} to add file synchronization
@@ -82,4 +83,9 @@ public final class ShadowIndexShard extends IndexShard {
public boolean allowsPrimaryPromotion() {
return false;
}
+
+ @Override
+ public TranslogStats translogStats() {
+ return null; // shadow engine has no translog
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index f893ec4d89d..68c552d4419 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -18,27 +18,13 @@
*/
package org.elasticsearch.index.shard;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.BitSetProducer;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.aliases.IndexAliasesService;
-import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException;
import org.elasticsearch.index.mapper.*;
-import org.elasticsearch.index.query.IndexQueryParserService;
-import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.index.translog.Translog;
import java.io.IOException;
@@ -53,20 +39,13 @@ import static org.elasticsearch.index.mapper.SourceToParse.source;
*/
public class TranslogRecoveryPerformer {
private final MapperService mapperService;
- private final IndexQueryParserService queryParserService;
- private final IndexAliasesService indexAliasesService;
- private final IndexCache indexCache;
private final ESLogger logger;
private final Map recoveredTypes = new HashMap<>();
private final ShardId shardId;
- protected TranslogRecoveryPerformer(ShardId shardId, MapperService mapperService, IndexQueryParserService queryParserService,
- IndexAliasesService indexAliasesService, IndexCache indexCache, ESLogger logger) {
+ protected TranslogRecoveryPerformer(ShardId shardId, MapperService mapperService, ESLogger logger) {
this.shardId = shardId;
this.mapperService = mapperService;
- this.queryParserService = queryParserService;
- this.indexAliasesService = indexAliasesService;
- this.indexCache = indexCache;
this.logger = logger;
}
@@ -145,19 +124,7 @@ public class TranslogRecoveryPerformer {
public void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates) {
try {
switch (operation.opType()) {
- case CREATE:
- Translog.Create create = (Translog.Create) operation;
- Engine.Create engineCreate = IndexShard.prepareCreate(docMapper(create.type()),
- source(create.source()).index(shardId.getIndex()).type(create.type()).id(create.id())
- .routing(create.routing()).parent(create.parent()).timestamp(create.timestamp()).ttl(create.ttl()),
- create.version(), create.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY);
- maybeAddMappingUpdate(engineCreate.type(), engineCreate.parsedDoc().dynamicMappingsUpdate(), engineCreate.id(), allowMappingUpdates);
- if (logger.isTraceEnabled()) {
- logger.trace("[translog] recover [create] op of [{}][{}]", create.type(), create.id());
- }
- engine.create(engineCreate);
- break;
- case SAVE:
+ case INDEX:
Translog.Index index = (Translog.Index) operation;
Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(index.source()).type(index.type()).id(index.id())
.routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()),
@@ -177,11 +144,6 @@ public class TranslogRecoveryPerformer {
engine.delete(new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.version(),
delete.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY, System.nanoTime(), false));
break;
- case DELETE_BY_QUERY:
- Translog.DeleteByQuery deleteByQuery = (Translog.DeleteByQuery) operation;
- engine.delete(prepareDeleteByQuery(queryParserService, mapperService, indexAliasesService, indexCache,
- deleteByQuery.source(), deleteByQuery.filteringAliases(), Engine.Operation.Origin.RECOVERY, deleteByQuery.types()));
- break;
default:
throw new IllegalStateException("No operation defined for [" + operation + "]");
}
@@ -206,38 +168,6 @@ public class TranslogRecoveryPerformer {
operationProcessed();
}
- private static Engine.DeleteByQuery prepareDeleteByQuery(IndexQueryParserService queryParserService, MapperService mapperService, IndexAliasesService indexAliasesService, IndexCache indexCache, BytesReference source, @Nullable String[] filteringAliases, Engine.Operation.Origin origin, String... types) {
- long startTime = System.nanoTime();
- if (types == null) {
- types = Strings.EMPTY_ARRAY;
- }
- Query query;
- try {
- query = queryParserService.parseQuery(source).query();
- } catch (ParsingException ex) {
- // for BWC we try to parse directly the query since pre 1.0.0.Beta2 we didn't require a top level query field
- if (queryParserService.getIndexCreatedVersion().onOrBefore(Version.V_1_0_0_Beta2)) {
- try {
- XContentParser parser = XContentHelper.createParser(source);
- ParsedQuery parse = queryParserService.parse(parser);
- query = parse.query();
- } catch (Throwable t) {
- ex.addSuppressed(t);
- throw ex;
- }
- } else {
- throw ex;
- }
- }
- Query searchFilter = mapperService.searchFilter(types);
- if (searchFilter != null) {
- query = Queries.filtered(query, searchFilter);
- }
-
- Query aliasFilter = indexAliasesService.aliasFilter(filteringAliases);
- BitSetProducer parentFilter = mapperService.hasNested() ? indexCache.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()) : null;
- return new Engine.DeleteByQuery(query, source, filteringAliases, aliasFilter, parentFilter, origin, startTime, types);
- }
/**
* Called once for every processed operation by this recovery performer.
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
index 1983c4e8ecf..68e50da1348 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
@@ -21,8 +21,6 @@ package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
@@ -40,8 +38,7 @@ public class BM25SimilarityProvider extends AbstractSimilarityProvider {
private final BM25Similarity similarity;
- @Inject
- public BM25SimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public BM25SimilarityProvider(String name, Settings settings) {
super(name);
float k1 = settings.getAsFloat("k1", 1.2f);
float b = settings.getAsFloat("b", 0.75f);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
index 7858cb132d6..d5caa4aab98 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
@@ -77,8 +77,7 @@ public class DFRSimilarityProvider extends AbstractSimilarityProvider {
private final DFRSimilarity similarity;
- @Inject
- public DFRSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public DFRSimilarityProvider(String name, Settings settings) {
super(name);
BasicModel basicModel = parseBasicModel(settings);
AfterEffect afterEffect = parseAfterEffect(settings);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/DefaultSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/DefaultSimilarityProvider.java
index 0f9feba952b..3acbd9821af 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/DefaultSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/DefaultSimilarityProvider.java
@@ -20,8 +20,6 @@
package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.DefaultSimilarity;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
@@ -37,8 +35,7 @@ public class DefaultSimilarityProvider extends AbstractSimilarityProvider {
private final DefaultSimilarity similarity = new DefaultSimilarity();
- @Inject
- public DefaultSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public DefaultSimilarityProvider(String name, Settings settings) {
super(name);
boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
this.similarity.setDiscountOverlaps(discountOverlaps);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
index 2f619c5615e..4b83bc838f2 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
@@ -28,8 +28,6 @@ import org.apache.lucene.search.similarities.LambdaDF;
import org.apache.lucene.search.similarities.LambdaTTF;
import org.apache.lucene.search.similarities.Normalization;
import org.apache.lucene.search.similarities.Similarity;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import java.util.HashMap;
@@ -67,8 +65,7 @@ public class IBSimilarityProvider extends AbstractSimilarityProvider {
private final IBSimilarity similarity;
- @Inject
- public IBSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public IBSimilarityProvider(String name, Settings settings) {
super(name);
Distribution distribution = parseDistribution(settings);
Lambda lambda = parseLambda(settings);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
index efea285639b..24494dc0b75 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
@@ -21,8 +21,6 @@ package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.LMDirichletSimilarity;
import org.apache.lucene.search.similarities.Similarity;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
@@ -38,8 +36,7 @@ public class LMDirichletSimilarityProvider extends AbstractSimilarityProvider {
private final LMDirichletSimilarity similarity;
- @Inject
- public LMDirichletSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public LMDirichletSimilarityProvider(String name, Settings settings) {
super(name);
float mu = settings.getAsFloat("mu", 2000f);
this.similarity = new LMDirichletSimilarity(mu);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
index 5d30b300d5c..3d5a40fc153 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
@@ -38,8 +38,7 @@ public class LMJelinekMercerSimilarityProvider extends AbstractSimilarityProvide
private final LMJelinekMercerSimilarity similarity;
- @Inject
- public LMJelinekMercerSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
+ public LMJelinekMercerSimilarityProvider(String name, Settings settings) {
super(name);
float lambda = settings.getAsFloat("lambda", 0.1f);
this.similarity = new LMJelinekMercerSimilarity(lambda);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/PreBuiltSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/PreBuiltSimilarityProvider.java
deleted file mode 100644
index 4b3f0ccf035..00000000000
--- a/core/src/main/java/org/elasticsearch/index/similarity/PreBuiltSimilarityProvider.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.similarity;
-
-import org.apache.lucene.search.similarities.Similarity;
-import org.elasticsearch.common.settings.Settings;
-
-/**
- * {@link SimilarityProvider} for pre-built Similarities
- */
-public class PreBuiltSimilarityProvider extends AbstractSimilarityProvider {
-
- public static class Factory implements SimilarityProvider.Factory {
-
- private final PreBuiltSimilarityProvider similarity;
-
- public Factory(String name, Similarity similarity) {
- this.similarity = new PreBuiltSimilarityProvider(name, similarity);
- }
-
- @Override
- public SimilarityProvider create(String name, Settings settings) {
- return similarity;
- }
-
- public String name() {
- return similarity.name();
- }
-
- public SimilarityProvider get() {
- return similarity;
- }
- }
-
- private final Similarity similarity;
-
- /**
- * Creates a new {@link PreBuiltSimilarityProvider} with the given name and given
- * pre-built Similarity
- *
- * @param name Name of the Provider
- * @param similarity Pre-built Similarity
- */
- public PreBuiltSimilarityProvider(String name, Similarity similarity) {
- super(name);
- this.similarity = similarity;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Similarity get() {
- return similarity;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java
index 6e03bcf0848..29312f2557b 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java
@@ -20,19 +20,18 @@
package org.elasticsearch.index.similarity;
import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Scopes;
-import org.elasticsearch.common.inject.assistedinject.FactoryProvider;
-import org.elasticsearch.common.inject.multibindings.MapBinder;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
import java.util.HashMap;
import java.util.Map;
+import java.util.function.BiFunction;
/**
* {@link SimilarityModule} is responsible gathering registered and configured {@link SimilarityProvider}
- * implementations and making them available through the {@link SimilarityLookupService} and {@link SimilarityService}.
+ * implementations and making them available through the {@link SimilarityService}.
*
- * New {@link SimilarityProvider} implementations can be registered through {@link #addSimilarity(String, Class)}
+ * New {@link SimilarityProvider} implementations can be registered through {@link #addSimilarity(String, BiFunction)}
* while existing Providers can be referenced through Settings under the {@link #SIMILARITY_SETTINGS_PREFIX} prefix
* along with the "type" value. For example, to reference the {@link BM25SimilarityProvider}, the configuration
* "index.similarity.my_similarity.type : "BM25" can be used.
@@ -42,16 +41,12 @@ public class SimilarityModule extends AbstractModule {
public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
private final Settings settings;
- private final Map> similarities = new HashMap<>();
+ private final Map> similarities = new HashMap<>();
+ private final Index index;
- public SimilarityModule(Settings settings) {
+ public SimilarityModule(Index index, Settings settings) {
this.settings = settings;
- addSimilarity("default", DefaultSimilarityProvider.class);
- addSimilarity("BM25", BM25SimilarityProvider.class);
- addSimilarity("DFR", DFRSimilarityProvider.class);
- addSimilarity("IB", IBSimilarityProvider.class);
- addSimilarity("LMDirichlet", LMDirichletSimilarityProvider.class);
- addSimilarity("LMJelinekMercer", LMJelinekMercerSimilarityProvider.class);
+ this.index = index;
}
/**
@@ -60,36 +55,16 @@ public class SimilarityModule extends AbstractModule {
* @param name Name of the SimilarityProvider
* @param similarity SimilarityProvider to register
*/
- public void addSimilarity(String name, Class similarity) {
+ public void addSimilarity(String name, BiFunction similarity) {
+ if (similarities.containsKey(name) || SimilarityService.BUILT_IN.containsKey(name)) {
+ throw new IllegalArgumentException("similarity for name: [" + name + " is already registered");
+ }
similarities.put(name, similarity);
}
@Override
protected void configure() {
- MapBinder similarityBinder =
- MapBinder.newMapBinder(binder(), String.class, SimilarityProvider.Factory.class);
-
- Map similaritySettings = settings.getGroups(SIMILARITY_SETTINGS_PREFIX);
- for (Map.Entry entry : similaritySettings.entrySet()) {
- String name = entry.getKey();
- Settings settings = entry.getValue();
-
- String typeName = settings.get("type");
- if (typeName == null) {
- throw new IllegalArgumentException("Similarity [" + name + "] must have an associated type");
- } else if (similarities.containsKey(typeName) == false) {
- throw new IllegalArgumentException("Unknown Similarity type [" + typeName + "] for [" + name + "]");
- }
- similarityBinder.addBinding(entry.getKey()).toProvider(FactoryProvider.newFactory(SimilarityProvider.Factory.class, similarities.get(typeName))).in(Scopes.SINGLETON);
- }
-
- for (PreBuiltSimilarityProvider.Factory factory : Similarities.listFactories()) {
- if (!similarities.containsKey(factory.name())) {
- similarityBinder.addBinding(factory.name()).toInstance(factory);
- }
- }
-
- bind(SimilarityLookupService.class).asEagerSingleton();
- bind(SimilarityService.class).asEagerSingleton();
+ SimilarityService service = new SimilarityService(index, settings, new HashMap<>(similarities));
+ bind(SimilarityService.class).toInstance(service);
}
}
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java
index 38f56af4514..6433181dd6d 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java
@@ -40,19 +40,4 @@ public interface SimilarityProvider {
* @return Provided {@link Similarity}
*/
Similarity get();
-
- /**
- * Factory for creating {@link SimilarityProvider} instances
- */
- public static interface Factory {
-
- /**
- * Creates a new {@link SimilarityProvider} instance
- *
- * @param name Name of the provider
- * @param settings Settings to be used by the Provider
- * @return {@link SimilarityProvider} instance created by the Factory
- */
- SimilarityProvider create(String name, Settings settings);
- }
}
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index 98faa87e94b..a77a2de4dff 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -25,55 +25,96 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
-import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiFunction;
+
/**
*
*/
public class SimilarityService extends AbstractIndexComponent {
- private final SimilarityLookupService similarityLookupService;
- private final MapperService mapperService;
-
- private final Similarity perFieldSimilarity;
-
+ public final static String DEFAULT_SIMILARITY = "default";
+ private final Similarity defaultSimilarity;
+ private final Similarity baseSimilarity;
+ private final Map similarities;
+ static final Map> DEFAULTS;
+ static final Map> BUILT_IN;
+ static {
+ Map> defaults = new HashMap<>();
+ Map> buildIn = new HashMap<>();
+ defaults.put("default", DefaultSimilarityProvider::new);
+ defaults.put("BM25", BM25SimilarityProvider::new);
+ buildIn.put("default", DefaultSimilarityProvider::new);
+ buildIn.put("BM25", BM25SimilarityProvider::new);
+ buildIn.put("DFR", DFRSimilarityProvider::new);
+ buildIn.put("IB", IBSimilarityProvider::new);
+ buildIn.put("LMDirichlet", LMDirichletSimilarityProvider::new);
+ buildIn.put("LMJelinekMercer", LMJelinekMercerSimilarityProvider::new);
+ DEFAULTS = Collections.unmodifiableMap(defaults);
+ BUILT_IN = Collections.unmodifiableMap(buildIn);
+ }
public SimilarityService(Index index) {
this(index, Settings.Builder.EMPTY_SETTINGS);
}
public SimilarityService(Index index, Settings settings) {
- this(index, settings, new SimilarityLookupService(index, settings), null);
+ this(index, settings, Collections.EMPTY_MAP);
}
@Inject
- public SimilarityService(Index index, @IndexSettings Settings indexSettings,
- final SimilarityLookupService similarityLookupService, final MapperService mapperService) {
+ public SimilarityService(Index index, @IndexSettings Settings indexSettings, Map> similarities) {
super(index, indexSettings);
- this.similarityLookupService = similarityLookupService;
- this.mapperService = mapperService;
-
- Similarity defaultSimilarity = similarityLookupService.similarity(SimilarityLookupService.DEFAULT_SIMILARITY).get();
+ Map providers = new HashMap<>(similarities.size());
+ Map similaritySettings = indexSettings.getGroups(SimilarityModule.SIMILARITY_SETTINGS_PREFIX);
+ for (Map.Entry entry : similaritySettings.entrySet()) {
+ String name = entry.getKey();
+ Settings settings = entry.getValue();
+ String typeName = settings.get("type");
+ if (typeName == null) {
+ throw new IllegalArgumentException("Similarity [" + name + "] must have an associated type");
+ } else if ((similarities.containsKey(typeName) || BUILT_IN.containsKey(typeName)) == false) {
+ throw new IllegalArgumentException("Unknown Similarity type [" + typeName + "] for [" + name + "]");
+ }
+ BiFunction factory = similarities.getOrDefault(typeName, BUILT_IN.get(typeName));
+ if (settings == null) {
+ settings = Settings.Builder.EMPTY_SETTINGS;
+ }
+ providers.put(name, factory.apply(name, settings));
+ }
+ addSimilarities(similaritySettings, providers, DEFAULTS);
+ this.similarities = providers;
+ defaultSimilarity = providers.get(SimilarityService.DEFAULT_SIMILARITY).get();
// Expert users can configure the base type as being different to default, but out-of-box we use default.
- Similarity baseSimilarity = (similarityLookupService.similarity("base") != null) ? similarityLookupService.similarity("base").get() :
- defaultSimilarity;
-
- this.perFieldSimilarity = (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, baseSimilarity, mapperService) :
+ baseSimilarity = (providers.get("base") != null) ? providers.get("base").get() :
defaultSimilarity;
}
- public Similarity similarity() {
- return perFieldSimilarity;
+ public Similarity similarity(MapperService mapperService) {
+ // TODO we can maybe factor out MapperService here entirely by introducing an interface for the lookup?
+ return (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, baseSimilarity, mapperService) :
+ defaultSimilarity;
}
- public SimilarityLookupService similarityLookupService() {
- return similarityLookupService;
+ private void addSimilarities(Map similaritySettings, Map providers, Map> similarities) {
+ for (Map.Entry> entry : similarities.entrySet()) {
+ String name = entry.getKey();
+ BiFunction factory = entry.getValue();
+ Settings settings = similaritySettings.get(name);
+ if (settings == null) {
+ settings = Settings.Builder.EMPTY_SETTINGS;
+ }
+ providers.put(name, factory.apply(name, settings));
+ }
}
- public MapperService mapperService() {
- return mapperService;
+ public SimilarityProvider getSimilarity(String name) {
+ return similarities.get(name);
}
static class PerFieldSimilarity extends PerFieldSimilarityWrapper {
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 5084895151b..4265d611fbf 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -23,11 +23,9 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TwoPhaseCommit;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
@@ -38,7 +36,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.BigArrays;
@@ -54,7 +51,6 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.EOFException;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.*;
@@ -189,99 +185,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
}
}
- /**
- * This method is used to upgarde a pre 2.0 translog structure to the new checkpoint based structure.
- * The {@link org.elasticsearch.index.translog.Translog.TranslogGeneration} in the given config is
- * used to determine the smallest file generation to upgrade. The procedure will travers the translog
- * directory to find all files that have a generation greater or equal to the translog generation and
- * renames the files to the new .tlog file format.
- *
- * For each of the files a ${filename}.ckp
- * file is written containing the size of the translog in bytes, it's ID and the number of operations. Since
- * these files are all relying on the pre 2.0 truncation feature where we read operations until hitting an {@link EOFException}
- * the number of operations are recoreded as -1 . Later once these files are opened for reading legacy readers will
- * allow for unknown number of operations and mimic the old behavior.
- *
- */
- public static void upgradeLegacyTranslog(ESLogger logger, TranslogConfig config) throws IOException {
- Path translogPath = config.getTranslogPath();
- TranslogGeneration translogGeneration = config.getTranslogGeneration();
- if (translogGeneration == null) {
- throw new IllegalArgumentException("TranslogGeneration must be set in order to upgrade");
- }
- if (translogGeneration.translogUUID != null) {
- throw new IllegalArgumentException("TranslogGeneration has a non-null UUID - index must have already been upgraded");
- }
- try {
- if (Checkpoint.read(translogPath.resolve(CHECKPOINT_FILE_NAME)) != null) {
- throw new IllegalStateException(CHECKPOINT_FILE_NAME + " file already present, translog is already upgraded");
- }
- } catch (NoSuchFileException | FileNotFoundException ex) {
- logger.debug("upgrading translog - no checkpoint found");
- }
- final Pattern parseLegacyIdPattern = Pattern.compile("^" + TRANSLOG_FILE_PREFIX + "(\\d+)((\\.recovering))?$"); // here we have to be lenient - nowhere else!
- try (DirectoryStream stream = Files.newDirectoryStream(translogPath, new DirectoryStream.Filter() {
- @Override
- public boolean accept(Path entry) throws IOException {
- Matcher matcher = parseLegacyIdPattern.matcher(entry.getFileName().toString());
- if (matcher.matches() == false) {
- Matcher newIdMatcher = PARSE_STRICT_ID_PATTERN.matcher(entry.getFileName().toString());
- return newIdMatcher.matches();
- } else {
- return true;
- }
- }
- })) {
- long latestGeneration = -1;
- List