+ * When running tests you have to pass it to the test runner like this:
+ *
+ * mvn test -Dtests.jvm.argline="-Djava.security.debug=access,failure" ...
+ *
* See
* Troubleshooting Security for information.
*/
@@ -99,8 +105,6 @@ final class Security {
* Can only happen once!
*/
static void configure(Environment environment) throws Exception {
- // set properties for jar locations
- setCodebaseProperties();
// enable security policy: union of template and environment-based paths, and possibly plugin permissions
Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment)));
@@ -121,70 +125,35 @@ final class Security {
selfTest();
}
- // mapping of jars to codebase properties
- // note that this is only read once, when policy is parsed.
- private static final Map SPECIAL_JARS;
- static {
- Map m = new IdentityHashMap<>();
- m.put(Pattern.compile(".*lucene-core-.*\\.jar$"), "es.security.jar.lucene.core");
- m.put(Pattern.compile(".*lucene-test-framework-.*\\.jar$"), "es.security.jar.lucene.testframework");
- m.put(Pattern.compile(".*randomizedtesting-runner-.*\\.jar$"), "es.security.jar.randomizedtesting.runner");
- m.put(Pattern.compile(".*junit4-ant-.*\\.jar$"), "es.security.jar.randomizedtesting.junit4");
- m.put(Pattern.compile(".*securemock-.*\\.jar$"), "es.security.jar.elasticsearch.securemock");
- SPECIAL_JARS = Collections.unmodifiableMap(m);
- }
-
- /**
- * Sets properties (codebase URLs) for policy files.
- * JAR locations are not fixed so we have to find the locations of
- * the ones we want.
- */
- @SuppressForbidden(reason = "proper use of URL")
- static void setCodebaseProperties() {
- for (URL url : JarHell.parseClassPath()) {
- for (Map.Entry e : SPECIAL_JARS.entrySet()) {
- if (e.getKey().matcher(url.getPath()).matches()) {
- String prop = e.getValue();
- if (System.getProperty(prop) != null) {
- throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop));
- }
- System.setProperty(prop, url.toString());
- }
- }
- }
- for (String prop : SPECIAL_JARS.values()) {
- if (System.getProperty(prop) == null) {
- System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all"
- }
- }
- }
-
/**
* Sets properties (codebase URLs) for policy files.
* we look for matching plugins and set URLs to fit
*/
@SuppressForbidden(reason = "proper use of URL")
- static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
- Map map = new HashMap<>();
+ static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
+ Map map = new HashMap<>();
if (Files.exists(environment.pluginsFile())) {
try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) {
for (Path plugin : stream) {
Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY);
if (Files.exists(policyFile)) {
- // parse the plugin's policy file into a set of permissions
- Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri()));
- PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain());
- // this method is supported with the specific implementation we use, but just check for safety.
- if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
- throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
- }
- // grant the permissions to each jar in the plugin
+ // first get a list of URLs for the plugins' jars:
+ // we resolve symlinks so map is keyed on the normalize codebase name
+ List codebases = new ArrayList<>();
try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
for (Path jar : jarStream) {
- if (map.put(jar.toUri().toURL().getFile(), permissions) != null) {
- // just be paranoid ok?
- throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
- }
+ codebases.add(jar.toRealPath().toUri().toURL());
+ }
+ }
+
+ // parse the plugin's policy file into a set of permissions
+ Policy policy = readPolicy(policyFile.toUri().toURL(), codebases.toArray(new URL[codebases.size()]));
+
+ // consult this policy for each of the plugin's jars:
+ for (URL url : codebases) {
+ if (map.put(url.getFile(), policy) != null) {
+ // just be paranoid ok?
+ throw new IllegalStateException("per-plugin permissions already granted for jar file: " + url);
}
}
}
@@ -194,6 +163,35 @@ final class Security {
return Collections.unmodifiableMap(map);
}
+ /**
+ * Reads and returns the specified {@code policyFile}.
+ *
+ * Resources (e.g. jar files and directories) listed in {@code codebases} location
+ * will be provided to the policy file via a system property of the short name:
+ * e.g. ${codebase.joda-convert-1.2.jar} would map to full URL.
+ */
+ @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
+ static Policy readPolicy(URL policyFile, URL codebases[]) {
+ try {
+ try {
+ // set codebase properties
+ for (URL url : codebases) {
+ String shortName = PathUtils.get(url.toURI()).getFileName().toString();
+ System.setProperty("codebase." + shortName, url.toString());
+ }
+ return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI()));
+ } finally {
+ // clear codebase properties
+ for (URL url : codebases) {
+ String shortName = PathUtils.get(url.toURI()).getFileName().toString();
+ System.clearProperty("codebase." + shortName);
+ }
+ }
+ } catch (NoSuchAlgorithmException | URISyntaxException e) {
+ throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e);
+ }
+ }
+
/** returns dynamic Permissions to configured paths */
static Permissions createPermissions(Environment environment) throws IOException {
Permissions policy = new Permissions();
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 17551012edb..10ef7bcb13c 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -143,7 +143,7 @@ public class TransportClient extends AbstractClient {
modules.add(new ClusterNameModule(this.settings));
modules.add(new ThreadPoolModule(threadPool));
modules.add(new TransportModule(this.settings));
- modules.add(new SearchModule(this.settings) {
+ modules.add(new SearchModule() {
@Override
protected void configure() {
// noop
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java
similarity index 50%
rename from core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java
rename to core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java
index e13661c814c..ed0a7f56b9c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java
@@ -17,30 +17,7 @@
* under the License.
*/
-package org.elasticsearch.index.query;
+package org.elasticsearch.cluster.action.shard;
-import java.io.IOException;
-
-/**
- * Parser for query filter
- * @deprecated use any query instead directly, possible since queries and filters are merged.
- */
-// TODO: remove when https://github.com/elastic/elasticsearch/issues/13326 is fixed
-@Deprecated
-public class QueryFilterParser implements QueryParser {
-
- @Override
- public String[] names() {
- return new String[]{QueryFilterBuilder.NAME};
- }
-
- @Override
- public QueryFilterBuilder fromXContent(QueryParseContext parseContext) throws IOException {
- return new QueryFilterBuilder(parseContext.parseInnerQueryBuilder());
- }
-
- @Override
- public QueryFilterBuilder getBuilderPrototype() {
- return QueryFilterBuilder.PROTOTYPE;
- }
-}
\ No newline at end of file
+public class NoOpShardStateActionListener implements ShardStateAction.Listener {
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 02867c930be..fc465ae283c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -77,27 +77,29 @@ public class ShardStateAction extends AbstractComponent {
transportService.registerRequestHandler(SHARD_FAILED_ACTION_NAME, ShardRoutingEntry::new, ThreadPool.Names.SAME, new ShardFailedTransportHandler());
}
- public void shardFailed(final ShardRouting shardRouting, final String indexUUID, final String message, @Nullable final Throwable failure) {
+ public void shardFailed(final ShardRouting shardRouting, final String indexUUID, final String message, @Nullable final Throwable failure, Listener listener) {
DiscoveryNode masterNode = clusterService.state().nodes().masterNode();
if (masterNode == null) {
logger.warn("can't send shard failed for {}, no master known.", shardRouting);
+ listener.onShardFailedNoMaster();
return;
}
- innerShardFailed(shardRouting, indexUUID, masterNode, message, failure);
+ innerShardFailed(shardRouting, indexUUID, masterNode, message, failure, listener);
}
- public void resendShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, @Nullable final Throwable failure) {
+ public void resendShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, @Nullable final Throwable failure, Listener listener) {
logger.trace("{} re-sending failed shard for {}, indexUUID [{}], reason [{}]", failure, shardRouting.shardId(), shardRouting, indexUUID, message);
- innerShardFailed(shardRouting, indexUUID, masterNode, message, failure);
+ innerShardFailed(shardRouting, indexUUID, masterNode, message, failure, listener);
}
- private void innerShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, final Throwable failure) {
+ private void innerShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, final Throwable failure, Listener listener) {
ShardRoutingEntry shardRoutingEntry = new ShardRoutingEntry(shardRouting, indexUUID, message, failure);
transportService.sendRequest(masterNode,
SHARD_FAILED_ACTION_NAME, shardRoutingEntry, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
@Override
public void handleException(TransportException exp) {
logger.warn("failed to send failed shard to {}", exp, masterNode);
+ listener.onShardFailedFailure(masterNode, exp);
}
});
}
@@ -284,4 +286,9 @@ public class ShardStateAction extends AbstractComponent {
return "" + shardRouting + ", indexUUID [" + indexUUID + "], message [" + message + "], failure [" + ExceptionsHelper.detailedMessage(failure) + "]";
}
}
+
+ public interface Listener {
+ default void onShardFailedNoMaster() {}
+ default void onShardFailedFailure(final DiscoveryNode master, final TransportException e) {}
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 251f45e2592..53309e77400 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -76,7 +76,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
}
/**
- * Translates the provided index expression into actual concrete indices.
+ * Translates the provided index expression into actual concrete indices, properly deduplicated.
*
* @param state the cluster state containing all the data to resolve to expressions to concrete indices
* @param options defines how the aliases or indices need to be resolved to concrete indices
@@ -94,7 +94,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
}
/**
- * Translates the provided index expression into actual concrete indices.
+ * Translates the provided index expression into actual concrete indices, properly deduplicated.
*
* @param state the cluster state containing all the data to resolve to expressions to concrete indices
* @param options defines how the aliases or indices need to be resolved to concrete indices
@@ -141,7 +141,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
}
}
- List concreteIndices = new ArrayList<>(expressions.size());
+ final Set concreteIndices = new HashSet<>(expressions.size());
for (String expression : expressions) {
AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
if (aliasOrIndex == null) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index cab86b60d44..f4c5ba513f0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -37,9 +37,9 @@ import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.threadpool.ThreadPool;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -66,9 +66,11 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
this.nodeIndexDeletedAction = nodeIndexDeletedAction;
}
- public void deleteIndex(final Request request, final Listener userListener) {
+ public void deleteIndices(final Request request, final Listener userListener) {
+ Collection indices = Arrays.asList(request.indices);
final DeleteIndexListener listener = new DeleteIndexListener(userListener);
- clusterService.submitStateUpdateTask("delete-index [" + request.index + "]", Priority.URGENT, new ClusterStateUpdateTask() {
+
+ clusterService.submitStateUpdateTask("delete-index " + indices, Priority.URGENT, new ClusterStateUpdateTask() {
@Override
public TimeValue timeout() {
@@ -82,34 +84,32 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
@Override
public ClusterState execute(final ClusterState currentState) {
- if (!currentState.metaData().hasConcreteIndex(request.index)) {
- throw new IndexNotFoundException(request.index);
- }
-
- logger.info("[{}] deleting index", request.index);
-
RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
- routingTableBuilder.remove(request.index);
+ MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData());
+ ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
- MetaData newMetaData = MetaData.builder(currentState.metaData())
- .remove(request.index)
- .build();
+ for (final String index: indices) {
+ if (!currentState.metaData().hasConcreteIndex(index)) {
+ throw new IndexNotFoundException(index);
+ }
- RoutingAllocation.Result routingResult = allocationService.reroute(
- ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).build());
-
- ClusterBlocks blocks = ClusterBlocks.builder().blocks(currentState.blocks()).removeIndexBlocks(request.index).build();
+ logger.debug("[{}] deleting index", index);
+ routingTableBuilder.remove(index);
+ clusterBlocksBuilder.removeIndexBlocks(index);
+ metaDataBuilder.remove(index);
+ }
// wait for events from all nodes that it has been removed from their respective metadata...
int count = currentState.nodes().size();
// add the notifications that the store was deleted from *data* nodes
count += currentState.nodes().dataNodes().size();
- final AtomicInteger counter = new AtomicInteger(count);
+ final AtomicInteger counter = new AtomicInteger(count * indices.size());
+
// this listener will be notified once we get back a notification based on the cluster state change below.
final NodeIndexDeletedAction.Listener nodeIndexDeleteListener = new NodeIndexDeletedAction.Listener() {
@Override
- public void onNodeIndexDeleted(String index, String nodeId) {
- if (index.equals(request.index)) {
+ public void onNodeIndexDeleted(String deleted, String nodeId) {
+ if (indices.contains(deleted)) {
if (counter.decrementAndGet() == 0) {
listener.onResponse(new Response(true));
nodeIndexDeletedAction.remove(this);
@@ -118,8 +118,8 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
}
@Override
- public void onNodeIndexStoreDeleted(String index, String nodeId) {
- if (index.equals(request.index)) {
+ public void onNodeIndexStoreDeleted(String deleted, String nodeId) {
+ if (indices.contains(deleted)) {
if (counter.decrementAndGet() == 0) {
listener.onResponse(new Response(true));
nodeIndexDeletedAction.remove(this);
@@ -128,15 +128,15 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
}
};
nodeIndexDeletedAction.add(nodeIndexDeleteListener);
-
- listener.future = threadPool.schedule(request.timeout, ThreadPool.Names.SAME, new Runnable() {
- @Override
- public void run() {
- listener.onResponse(new Response(false));
- nodeIndexDeletedAction.remove(nodeIndexDeleteListener);
- }
+ listener.future = threadPool.schedule(request.timeout, ThreadPool.Names.SAME, () -> {
+ listener.onResponse(new Response(false));
+ nodeIndexDeletedAction.remove(nodeIndexDeleteListener);
});
+ MetaData newMetaData = metaDataBuilder.build();
+ ClusterBlocks blocks = clusterBlocksBuilder.build();
+ RoutingAllocation.Result routingResult = allocationService.reroute(
+ ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).build());
return ClusterState.builder(currentState).routingResult(routingResult).metaData(newMetaData).blocks(blocks).build();
}
@@ -173,7 +173,6 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
}
}
-
public interface Listener {
void onResponse(Response response);
@@ -183,13 +182,13 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
public static class Request {
- final String index;
+ final String[] indices;
TimeValue timeout = TimeValue.timeValueSeconds(10);
TimeValue masterTimeout = MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
- public Request(String index) {
- this.index = index;
+ public Request(String[] indices) {
+ this.indices = indices;
}
public Request timeout(TimeValue timeout) {
diff --git a/core/src/main/java/org/elasticsearch/common/cache/Cache.java b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
index a686ecc9645..595ac088140 100644
--- a/core/src/main/java/org/elasticsearch/common/cache/Cache.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
@@ -25,12 +25,11 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.BiFunction;
import java.util.function.ToLongBiFunction;
/**
@@ -175,7 +174,7 @@ public class Cache {
ReleasableLock readLock = new ReleasableLock(segmentLock.readLock());
ReleasableLock writeLock = new ReleasableLock(segmentLock.writeLock());
- Map>> map = new HashMap<>();
+ Map>> map = new HashMap<>();
SegmentStats segmentStats = new SegmentStats();
@@ -187,20 +186,28 @@ public class Cache {
* @return the entry if there was one, otherwise null
*/
Entry get(K key, long now) {
- Future> future;
+ CompletableFuture> future;
Entry entry = null;
try (ReleasableLock ignored = readLock.acquire()) {
future = map.get(key);
}
if (future != null) {
- segmentStats.hit();
- try {
- entry = future.get();
- entry.accessTime = now;
- } catch (ExecutionException | InterruptedException e) {
- throw new IllegalStateException("future should be a completedFuture for which get should not throw", e);
- }
- } else {
+ try {
+ entry = future.handle((ok, ex) -> {
+ if (ok != null) {
+ segmentStats.hit();
+ ok.accessTime = now;
+ return ok;
+ } else {
+ segmentStats.miss();
+ return null;
+ }
+ }).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ else {
segmentStats.miss();
}
return entry;
@@ -216,11 +223,19 @@ public class Cache {
*/
Tuple, Entry> put(K key, V value, long now) {
Entry entry = new Entry<>(key, value, now);
- Entry existing;
+ Entry existing = null;
try (ReleasableLock ignored = writeLock.acquire()) {
try {
- Future> future = map.put(key, CompletableFuture.completedFuture(entry));
- existing = future != null ? future.get() : null;
+ CompletableFuture> future = map.put(key, CompletableFuture.completedFuture(entry));
+ if (future != null) {
+ existing = future.handle((ok, ex) -> {
+ if (ok != null) {
+ return ok;
+ } else {
+ return null;
+ }
+ }).get();
+ }
} catch (ExecutionException | InterruptedException e) {
throw new IllegalStateException("future should be a completedFuture for which get should not throw", e);
}
@@ -235,17 +250,23 @@ public class Cache {
* @return the removed entry if there was one, otherwise null
*/
Entry remove(K key) {
- Future> future;
+ CompletableFuture> future;
Entry entry = null;
try (ReleasableLock ignored = writeLock.acquire()) {
future = map.remove(key);
}
if (future != null) {
- segmentStats.eviction();
try {
- entry = future.get();
+ entry = future.handle((ok, ex) -> {
+ if (ok != null) {
+ segmentStats.eviction();
+ return ok;
+ } else {
+ return null;
+ }
+ }).get();
} catch (ExecutionException | InterruptedException e) {
- throw new IllegalStateException("future should be a completedFuture for which get should not throw", e);
+ throw new IllegalStateException(e);
}
}
return entry;
@@ -327,39 +348,57 @@ public class Cache {
// the segment lock; to do this, we atomically put a future in the map that can load the value, and then
// get the value from this future on the thread that won the race to place the future into the segment map
CacheSegment segment = getCacheSegment(key);
- Future> future;
- FutureTask> task = new FutureTask<>(() -> new Entry<>(key, loader.load(key), now));
+ CompletableFuture> future;
+ CompletableFuture> completableFuture = new CompletableFuture<>();
+
try (ReleasableLock ignored = segment.writeLock.acquire()) {
- future = segment.map.putIfAbsent(key, task);
- }
- if (future == null) {
- future = task;
- task.run();
+ future = segment.map.putIfAbsent(key, completableFuture);
}
- Entry entry;
- try {
- entry = future.get();
- } catch (ExecutionException | InterruptedException e) {
- // if the future ended exceptionally, we do not want to pollute the cache
- // however, we have to take care to ensure that the polluted entry has not already been replaced
- try (ReleasableLock ignored = segment.writeLock.acquire()) {
- Future> sanity = segment.map.get(key);
- try {
- sanity.get();
- } catch (ExecutionException | InterruptedException gotcha) {
- segment.map.remove(key);
+ BiFunction, Throwable, ? extends V> handler = (ok, ex) -> {
+ if (ok != null) {
+ try (ReleasableLock ignored = lruLock.acquire()) {
+ promote(ok, now);
}
+ return ok.value;
+ } else {
+ try (ReleasableLock ignored = segment.writeLock.acquire()) {
+ CompletableFuture> sanity = segment.map.get(key);
+ if (sanity != null && sanity.isCompletedExceptionally()) {
+ segment.map.remove(key);
+ }
+ }
+ return null;
}
- throw (e instanceof ExecutionException) ? (ExecutionException)e : new ExecutionException(e);
+ };
+
+ CompletableFuture completableValue;
+ if (future == null) {
+ future = completableFuture;
+ completableValue = future.handle(handler);
+ V loaded;
+ try {
+ loaded = loader.load(key);
+ } catch (Exception e) {
+ future.completeExceptionally(e);
+ throw new ExecutionException(e);
+ }
+ if (loaded == null) {
+ NullPointerException npe = new NullPointerException("loader returned a null value");
+ future.completeExceptionally(npe);
+ throw new ExecutionException(npe);
+ } else {
+ future.complete(new Entry<>(key, loaded, now));
+ }
+ } else {
+ completableValue = future.handle(handler);
}
- if (entry.value == null) {
- throw new ExecutionException(new NullPointerException("loader returned a null value"));
+
+ try {
+ value = completableValue.get();
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
}
- try (ReleasableLock ignored = lruLock.acquire()) {
- promote(entry, now);
- }
- value = entry.value;
}
return value;
}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index 1a3b50cc339..7130537fceb 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -20,8 +20,8 @@
package org.elasticsearch.common.geo;
import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.XGeoHashUtils;
-import org.apache.lucene.util.XGeoUtils;
+import org.apache.lucene.util.GeoHashUtils;
+import org.apache.lucene.util.GeoUtils;
/**
*
@@ -81,14 +81,14 @@ public final class GeoPoint {
}
public GeoPoint resetFromIndexHash(long hash) {
- lon = XGeoUtils.mortonUnhashLon(hash);
- lat = XGeoUtils.mortonUnhashLat(hash);
+ lon = GeoUtils.mortonUnhashLon(hash);
+ lat = GeoUtils.mortonUnhashLat(hash);
return this;
}
public GeoPoint resetFromGeoHash(String geohash) {
- final long hash = XGeoHashUtils.mortonEncode(geohash);
- return this.reset(XGeoUtils.mortonUnhashLat(hash), XGeoUtils.mortonUnhashLon(hash));
+ final long hash = GeoHashUtils.mortonEncode(geohash);
+ return this.reset(GeoUtils.mortonUnhashLat(hash), GeoUtils.mortonUnhashLon(hash));
}
public GeoPoint resetFromGeoHash(long geohashLong) {
@@ -113,11 +113,11 @@ public final class GeoPoint {
}
public final String geohash() {
- return XGeoHashUtils.stringEncode(lon, lat);
+ return GeoHashUtils.stringEncode(lon, lat);
}
public final String getGeohash() {
- return XGeoHashUtils.stringEncode(lon, lat);
+ return GeoHashUtils.stringEncode(lon, lat);
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java b/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java
index 695db015eda..64c657c8b6f 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java
@@ -28,11 +28,7 @@ import java.util.Collection;
import java.util.List;
/**
- * Overrides bounding box logic in ShapeCollection base class to comply with
- * OGC OpenGIS Abstract Specification: An Object Model for Interoperable Geoprocessing.
- *
- * NOTE: This algorithm is O(N) and can possibly be improved O(log n) using an internal R*-Tree
- * data structure for a collection of bounding boxes
+ * Extends spatial4j ShapeCollection for points_only shape indexing support
*/
public class XShapeCollection extends ShapeCollection {
@@ -49,42 +45,4 @@ public class XShapeCollection extends ShapeCollection {
public void setPointsOnly(boolean pointsOnly) {
this.pointsOnly = pointsOnly;
}
-
- @Override
- protected Rectangle computeBoundingBox(Collection shapes, SpatialContext ctx) {
- Rectangle retBox = shapes.iterator().next().getBoundingBox();
- for (Shape geom : shapes) {
- retBox = expandBBox(retBox, geom.getBoundingBox());
- }
- return retBox;
- }
-
- /**
- * Spatial4J shapes have no knowledge of directed edges. For this reason, a bounding box
- * that wraps the dateline can have a min longitude that is mathematically > than the
- * Rectangles' minX value. This is an issue for geometric collections (e.g., MultiPolygon
- * and ShapeCollection) Until geometry logic can be cleaned up in Spatial4J, ES provides
- * the following expansion algorithm for GeometryCollections
- */
- private Rectangle expandBBox(Rectangle bbox, Rectangle expand) {
- if (bbox.equals(expand) || bbox.equals(SpatialContext.GEO.getWorldBounds())) {
- return bbox;
- }
-
- double minX = bbox.getMinX();
- double eMinX = expand.getMinX();
- double maxX = bbox.getMaxX();
- double eMaxX = expand.getMaxX();
- double minY = bbox.getMinY();
- double eMinY = expand.getMinY();
- double maxY = bbox.getMaxY();
- double eMaxY = expand.getMaxY();
-
- bbox.reset(Math.min(Math.min(minX, maxX), Math.min(eMinX, eMaxX)),
- Math.max(Math.max(minX, maxX), Math.max(eMinX, eMaxX)),
- Math.min(Math.min(minY, maxY), Math.min(eMinY, eMaxY)),
- Math.max(Math.max(minY, maxY), Math.max(eMinY, eMaxY)));
-
- return bbox;
- }
}
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 25c0743b938..16a9796d8b6 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -60,9 +60,9 @@ public class Lucene {
public static final Version VERSION = Version.LATEST;
public static final Version ANALYZER_VERSION = VERSION;
public static final Version QUERYPARSER_VERSION = VERSION;
- public static final String LATEST_DOC_VALUES_FORMAT = "Lucene50";
+ public static final String LATEST_DOC_VALUES_FORMAT = "Lucene54";
public static final String LATEST_POSTINGS_FORMAT = "Lucene50";
- public static final String LATEST_CODEC = "Lucene53";
+ public static final String LATEST_CODEC = "Lucene54";
static {
Deprecated annotation = PostingsFormat.forName(LATEST_POSTINGS_FORMAT).getClass().getAnnotation(Deprecated.class);
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index 13eb86f1ce4..980543d45e6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -87,4 +87,10 @@ public interface Discovery extends LifecycleComponent {
super(msg, cause, args);
}
}
+
+ /**
+ * @return stats about the discovery
+ */
+ DiscoveryStats stats();
+
}
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
new file mode 100644
index 00000000000..dcd75b07651
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
+
+import java.io.IOException;
+
+public class DiscoveryStats implements Streamable, ToXContent {
+
+ @Nullable
+ private PendingClusterStateStats queueStats;
+
+ public DiscoveryStats(PendingClusterStateStats queueStats) {
+ this.queueStats = queueStats;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(Fields.DISCOVERY);
+
+ if (queueStats != null ){
+ queueStats.toXContent(builder, params);
+ }
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ public void readFrom(StreamInput in) throws IOException {
+ if (in.readBoolean()) {
+ queueStats = new PendingClusterStateStats();
+ queueStats.readFrom(in);
+ }
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ if (queueStats != null ) {
+ out.writeBoolean(true);
+ queueStats.writeTo(out);
+ }else{
+ out.writeBoolean(false);
+ }
+ }
+
+ static final class Fields {
+ static final XContentBuilderString DISCOVERY = new XContentBuilderString("discovery");
+ }
+
+ public PendingClusterStateStats getQueueStats() {
+ return queueStats;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 0979554b0c4..dd001294b97 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -316,6 +316,11 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem
}
}
+ @Override
+ public DiscoveryStats stats() {
+ return new DiscoveryStats(null);
+ }
+
private LocalDiscovery[] members() {
ClusterGroup clusterGroup = clusterGroups.get(clusterName);
if (clusterGroup == null) {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 9952f65a1b8..2b126a98ce2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -43,6 +43,8 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.DiscoveryStats;
+import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats;
import org.elasticsearch.discovery.InitialStateDiscoveryListener;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.discovery.zen.fd.MasterFaultDetection;
@@ -337,6 +339,12 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen
}
}
+ @Override
+ public DiscoveryStats stats() {
+ PendingClusterStateStats queueStats = publishClusterState.pendingStatesQueue().stats();
+ return new DiscoveryStats(queueStats);
+ }
+
/**
* returns true if zen discovery is started and there is a currently a background thread active for (re)joining
* the cluster used for testing.
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java
new file mode 100644
index 00000000000..44265b0e481
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.zen.publish;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+
+import java.io.IOException;
+
+/**
+ * Class encapsulating stats about the PendingClusterStatsQueue
+ */
+public class PendingClusterStateStats implements Streamable, ToXContent {
+
+ private int total;
+ private int pending;
+ private int committed;
+
+ public PendingClusterStateStats() {
+
+ }
+
+ public PendingClusterStateStats(int total, int pending, int committed) {
+ this.total = total;
+ this.pending = pending;
+ this.committed = committed;
+ }
+
+ public int getCommitted() {
+ return committed;
+ }
+
+ public int getPending() {
+ return pending;
+ }
+
+ public int getTotal() {
+ return total;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(Fields.QUEUE);
+ builder.field(Fields.TOTAL, total);
+ builder.field(Fields.PENDING, pending);
+ builder.field(Fields.COMMITTED, committed);
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ public void readFrom(StreamInput in) throws IOException {
+ total = in.readVInt();
+ pending = in.readVInt();
+ committed = in.readVInt();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVInt(total);
+ out.writeVInt(pending);
+ out.writeVInt(committed);
+ }
+
+ static final class Fields {
+ static final XContentBuilderString QUEUE = new XContentBuilderString("cluster_state_queue");
+ static final XContentBuilderString TOTAL = new XContentBuilderString("total");
+ static final XContentBuilderString PENDING = new XContentBuilderString("pending");
+ static final XContentBuilderString COMMITTED = new XContentBuilderString("committed");
+ }
+
+ @Override
+ public String toString() {
+ return "PendingClusterStateStats(total=" + total + ", pending=" + pending + ", committed=" + committed + ")";
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
index e3550e657fc..2f444f50288 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
@@ -283,4 +283,17 @@ public class PendingClusterStatesQueue {
}
}
+ public synchronized PendingClusterStateStats stats() {
+
+ // calculate committed cluster state
+ int committed = 0;
+ for (ClusterStateContext clusterStatsContext : pendingStates) {
+ if (clusterStatsContext.committed()) {
+ committed += 1;
+ }
+ }
+
+ return new PendingClusterStateStats(pendingStates.size(), pendingStates.size() - committed, committed);
+ }
+
}
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index 3d07ca632b5..cb0f3113ff5 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -19,9 +19,14 @@
package org.elasticsearch.index;
+import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.util.Providers;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.cache.IndexCache;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.cache.query.QueryCache;
+import org.elasticsearch.index.cache.query.index.IndexQueryCache;
+import org.elasticsearch.index.cache.query.none.NoneQueryCache;
import org.elasticsearch.index.engine.EngineFactory;
import org.elasticsearch.index.engine.InternalEngineFactory;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
@@ -33,6 +38,8 @@ import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreConfig;
+import org.elasticsearch.indices.IndicesWarmer;
+import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import java.util.*;
import java.util.function.BiFunction;
@@ -50,25 +57,37 @@ import java.util.function.Consumer;
*
Settings update listener - Custom settings update listener can be registered via {@link #addIndexSettingsListener(Consumer)}
*
*/
-public class IndexModule extends AbstractModule {
+public final class IndexModule extends AbstractModule {
public static final String STORE_TYPE = "index.store.type";
public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
+ public static final String INDEX_QUERY_CACHE = "index";
+ public static final String NONE_QUERY_CACHE = "none";
+ public static final String QUERY_CACHE_TYPE = "index.queries.cache.type";
+ // for test purposes only
+ public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything";
private final IndexSettings indexSettings;
private final IndexStoreConfig indexStoreConfig;
+ private final IndicesQueryCache indicesQueryCache;
// pkg private so tests can mock
Class engineFactoryImpl = InternalEngineFactory.class;
- Class indexSearcherWrapper = null;
+ private SetOnce indexSearcherWrapper = new SetOnce<>();
private final Set> settingsConsumers = new HashSet<>();
private final Set indexEventListeners = new HashSet<>();
private IndexEventListener listener;
private final Map> similarities = new HashMap<>();
private final Map> storeTypes = new HashMap<>();
+ private final Map> queryCaches = new HashMap<>();
+ private IndicesWarmer indicesWarmer;
- public IndexModule(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig) {
+ public IndexModule(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig, IndicesQueryCache indicesQueryCache, IndicesWarmer warmer) {
this.indexStoreConfig = indexStoreConfig;
this.indexSettings = indexSettings;
+ this.indicesQueryCache = indicesQueryCache;
+ this.indicesWarmer = warmer;
+ registerQueryCache(INDEX_QUERY_CACHE, IndexQueryCache::new);
+ registerQueryCache(NONE_QUERY_CACHE, (a, b) -> new NoneQueryCache(a));
}
/**
@@ -155,6 +174,28 @@ public class IndexModule extends AbstractModule {
similarities.put(name, similarity);
}
+ /**
+ * Registers a {@link QueryCache} provider for a given name
+ * @param name the providers / caches name
+ * @param provider the provider instance
+ */
+ public void registerQueryCache(String name, BiFunction provider) {
+ if (provider == null) {
+ throw new IllegalArgumentException("provider must not be null");
+ }
+ if (queryCaches.containsKey(name)) {
+ throw new IllegalArgumentException("Can't register the same [query_cache] more than once for [" + name + "]");
+ }
+ queryCaches.put(name, provider);
+ }
+
+ /**
+ * Sets a {@link org.elasticsearch.index.IndexModule.IndexSearcherWrapperFactory} that is called once the IndexService is fully constructed.
+ * Note: this method can only be called once per index. Multiple wrappers are not supported.
+ */
+ public void setSearcherWrapper(IndexSearcherWrapperFactory indexSearcherWrapperFactory) {
+ this.indexSearcherWrapper.set(indexSearcherWrapperFactory);
+ }
public IndexEventListener freeze() {
// TODO somehow we need to make this pkg private...
@@ -176,11 +217,7 @@ public class IndexModule extends AbstractModule {
@Override
protected void configure() {
bind(EngineFactory.class).to(engineFactoryImpl).asEagerSingleton();
- if (indexSearcherWrapper == null) {
- bind(IndexSearcherWrapper.class).toProvider(Providers.of(null));
- } else {
- bind(IndexSearcherWrapper.class).to(indexSearcherWrapper).asEagerSingleton();
- }
+ bind(IndexSearcherWrapperFactory.class).toInstance(indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get());
bind(IndexEventListener.class).toInstance(freeze());
bind(IndexService.class).asEagerSingleton();
bind(IndexServicesProvider.class).asEagerSingleton();
@@ -203,6 +240,15 @@ public class IndexModule extends AbstractModule {
throw new IllegalStateException("store must not be null");
}
}
+
+ final String queryCacheType = settings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
+ BiFunction queryCacheProvider = queryCaches.get(queryCacheType);
+ BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, indicesWarmer);
+ QueryCache queryCache = queryCacheProvider.apply(settings, indicesQueryCache);
+ IndexCache indexCache = new IndexCache(settings, queryCache, bitsetFilterCache);
+ bind(QueryCache.class).toInstance(queryCache);
+ bind(IndexCache.class).toInstance(indexCache);
+ bind(BitsetFilterCache.class).toInstance(bitsetFilterCache);
bind(IndexStore.class).toInstance(store);
bind(SimilarityService.class).toInstance(new SimilarityService(settings, similarities));
}
@@ -224,4 +270,14 @@ public class IndexModule extends AbstractModule {
return getSettingsKey().equals(setting);
}
}
+
+ /**
+ * Factory for creating new {@link IndexSearcherWrapper} instances
+ */
+ public interface IndexSearcherWrapperFactory {
+ /**
+ * Returns a new IndexSearcherWrapper. This method is called once per index per node
+ */
+ IndexSearcherWrapper newWrapper(final IndexService indexService);
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 70ff320ebbe..02cad6212df 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -75,6 +75,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
private final IndicesService indicesServices;
private final IndexServicesProvider indexServicesProvider;
private final IndexStore indexStore;
+ private final IndexSearcherWrapper searcherWrapper;
private volatile Map shards = emptyMap();
private final AtomicBoolean closed = new AtomicBoolean(false);
private final AtomicBoolean deleted = new AtomicBoolean(false);
@@ -88,7 +89,8 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
IndicesService indicesServices,
IndexServicesProvider indexServicesProvider,
IndexStore indexStore,
- IndexEventListener eventListener) {
+ IndexEventListener eventListener,
+ IndexModule.IndexSearcherWrapperFactory wrapperFactory) {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = analysisService;
@@ -101,6 +103,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
bitSetFilterCache.setListener(new BitsetCacheListener(this));
+ this.searcherWrapper = wrapperFactory.newWrapper(this);
}
public int numberOfShards() {
@@ -265,9 +268,9 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
(primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> indexServicesProvider.getIndicesQueryCache().onClose(shardId)));
if (useShadowEngine(primary, indexSettings)) {
- indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, indexServicesProvider);
+ indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, searcherWrapper, indexServicesProvider);
} else {
- indexShard = new IndexShard(shardId, this.indexSettings, path, store, indexServicesProvider);
+ indexShard = new IndexShard(shardId, this.indexSettings, path, store, searcherWrapper, indexServicesProvider);
}
eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
diff --git a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
index d61c911ab7d..1043d581af3 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java
@@ -56,12 +56,11 @@ public final class IndexServicesProvider {
private final SimilarityService similarityService;
private final EngineFactory factory;
private final BigArrays bigArrays;
- private final IndexSearcherWrapper indexSearcherWrapper;
private final IndexingMemoryController indexingMemoryController;
private final IndexEventListener listener;
@Inject
- public IndexServicesProvider(IndexEventListener listener, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper, IndexingMemoryController indexingMemoryController) {
+ public IndexServicesProvider(IndexEventListener listener, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, IndexingMemoryController indexingMemoryController) {
this.listener = listener;
this.threadPool = threadPool;
this.mapperService = mapperService;
@@ -75,7 +74,6 @@ public final class IndexServicesProvider {
this.similarityService = similarityService;
this.factory = factory;
this.bigArrays = bigArrays;
- this.indexSearcherWrapper = indexSearcherWrapper;
this.indexingMemoryController = indexingMemoryController;
}
@@ -126,13 +124,7 @@ public final class IndexServicesProvider {
return factory;
}
- public BigArrays getBigArrays() {
- return bigArrays;
- }
-
- public IndexSearcherWrapper getIndexSearcherWrapper() {
- return indexSearcherWrapper;
- }
+ public BigArrays getBigArrays() { return bigArrays; }
public IndexingMemoryController getIndexingMemoryController() {
return indexingMemoryController;
diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
index 67a7e717021..61733f24695 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -20,7 +20,6 @@
package org.elasticsearch.index.cache;
import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -37,7 +36,6 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
private final QueryCache queryCache;
private final BitsetFilterCache bitsetFilterCache;
- @Inject
public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) {
super(indexSettings);
this.queryCache = queryCache;
diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java
deleted file mode 100644
index 86e20490fa1..00000000000
--- a/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.cache.query.QueryCache;
-import org.elasticsearch.index.cache.query.index.IndexQueryCache;
-import org.elasticsearch.index.cache.query.none.NoneQueryCache;
-
-public class IndexCacheModule extends AbstractModule {
-
- public static final String INDEX_QUERY_CACHE = "index";
- public static final String NONE_QUERY_CACHE = "none";
- public static final String QUERY_CACHE_TYPE = "index.queries.cache.type";
- // for test purposes only
- public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything";
-
- private final Settings indexSettings;
- private final ExtensionPoint.SelectedType queryCaches;
-
- public IndexCacheModule(Settings settings) {
- this.indexSettings = settings;
- this.queryCaches = new ExtensionPoint.SelectedType<>("query_cache", QueryCache.class);
-
- registerQueryCache(INDEX_QUERY_CACHE, IndexQueryCache.class);
- registerQueryCache(NONE_QUERY_CACHE, NoneQueryCache.class);
- }
-
- public void registerQueryCache(String name, Class clazz) {
- queryCaches.registerExtension(name, clazz);
- }
-
- @Override
- protected void configure() {
- queryCaches.bindType(binder(), indexSettings, QUERY_CACHE_TYPE, INDEX_QUERY_CACHE);
- bind(BitsetFilterCache.class).asEagerSingleton();
- bind(IndexCache.class).asEagerSingleton();
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 994033273a3..7ef4c5b3da1 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -32,16 +32,15 @@ import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.BitSet;
import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.cache.Cache;
import org.elasticsearch.common.cache.CacheBuilder;
import org.elasticsearch.common.cache.RemovalListener;
import org.elasticsearch.common.cache.RemovalNotification;
-import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.object.ObjectMapper;
@@ -50,7 +49,6 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.IndicesWarmer.TerminationHandle;
-import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
@@ -68,7 +66,7 @@ import java.util.concurrent.Executor;
* and require that it should always be around should use this cache, otherwise the
* {@link org.elasticsearch.index.cache.query.QueryCache} should be used instead.
*/
-public class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener>, Closeable {
+public final class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener>, Closeable {
public static final String LOAD_RANDOM_ACCESS_FILTERS_EAGERLY = "index.load_fixed_bitset_filters_eagerly";
private static final Listener DEFAULT_NOOP_LISTENER = new Listener() {
@@ -85,24 +83,17 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
private final Cache> loadedFilters;
private volatile Listener listener = DEFAULT_NOOP_LISTENER;
private final BitSetProducerWarmer warmer;
+ private final IndicesWarmer indicesWarmer;
- private IndicesWarmer indicesWarmer;
-
- @Inject
- public BitsetFilterCache(IndexSettings indexSettings) {
+ public BitsetFilterCache(IndexSettings indexSettings, IndicesWarmer indicesWarmer) {
super(indexSettings);
this.loadRandomAccessFiltersEagerly = this.indexSettings.getSettings().getAsBoolean(LOAD_RANDOM_ACCESS_FILTERS_EAGERLY, true);
this.loadedFilters = CacheBuilder.>builder().removalListener(this).build();
this.warmer = new BitSetProducerWarmer();
- }
-
-
- @Inject(optional = true)
- public void setIndicesWarmer(IndicesWarmer indicesWarmer) {
this.indicesWarmer = indicesWarmer;
indicesWarmer.addListener(warmer);
}
-
+
/**
* Sets a listener that is invoked for all subsequent cache and removal events.
* @throws IllegalStateException if the listener is set more than once
@@ -129,10 +120,11 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
@Override
public void close() {
- if (indicesWarmer != null) {
+ try {
indicesWarmer.removeListener(warmer);
+ } finally {
+ clear("close");
}
- clear("close");
}
public void clear(String reason) {
@@ -229,10 +221,10 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
}
}
- final class BitSetProducerWarmer extends IndicesWarmer.Listener {
+ final class BitSetProducerWarmer implements IndicesWarmer.Listener {
@Override
- public IndicesWarmer.TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, IndicesWarmer.WarmerContext context, ThreadPool threadPool) {
+ public IndicesWarmer.TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) {
if (!loadRandomAccessFiltersEagerly) {
return TerminationHandle.NO_WAIT;
}
@@ -258,9 +250,9 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
warmUp.add(Queries.newNonNestedFilter());
}
- final Executor executor = threadPool.executor(executor());
- final CountDownLatch latch = new CountDownLatch(context.searcher().reader().leaves().size() * warmUp.size());
- for (final LeafReaderContext ctx : context.searcher().reader().leaves()) {
+ final Executor executor = indicesWarmer.getExecutor();
+ final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size() * warmUp.size());
+ for (final LeafReaderContext ctx : searcher.reader().leaves()) {
for (final Query filterToWarm : warmUp) {
executor.execute(() -> {
try {
@@ -281,7 +273,7 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea
}
@Override
- public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, IndicesWarmer.WarmerContext context, ThreadPool threadPool) {
+ public TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher) {
return TerminationHandle.NO_WAIT;
}
diff --git a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
index 9a15d853eda..f7c53cf20f0 100644
--- a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.codecs.lucene53.Lucene53Codec;
+import org.apache.lucene.codecs.lucene54.Lucene54Codec;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@@ -56,8 +56,8 @@ public class CodecService extends AbstractIndexComponent {
this.mapperService = mapperService;
MapBuilder codecs = MapBuilder.newMapBuilder();
if (mapperService == null) {
- codecs.put(DEFAULT_CODEC, new Lucene53Codec());
- codecs.put(BEST_COMPRESSION_CODEC, new Lucene53Codec(Mode.BEST_COMPRESSION));
+ codecs.put(DEFAULT_CODEC, new Lucene54Codec());
+ codecs.put(BEST_COMPRESSION_CODEC, new Lucene54Codec(Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC,
new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
diff --git a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java
index b8e44bdadb6..b504c4c21c5 100644
--- a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java
+++ b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java
@@ -22,7 +22,7 @@ package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene53.Lucene53Codec;
+import org.apache.lucene.codecs.lucene54.Lucene54Codec;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.MappedFieldType;
@@ -38,7 +38,7 @@ import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
* configured for a specific field the default postings format is used.
*/
// LUCENE UPGRADE: make sure to move to a new codec depending on the lucene version
-public class PerFieldMappingPostingFormatCodec extends Lucene53Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene54Codec {
private final ESLogger logger;
private final MapperService mapperService;
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 81f27097c81..46f144ea5f2 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -1056,4 +1056,18 @@ public abstract class Engine implements Closeable {
public long getLastWriteNanos() {
return this.lastWriteNanos;
}
+
+ /**
+ * Called for each new opened engine searcher to warm new segments
+ * @see EngineConfig#getWarmer()
+ */
+ public interface Warmer {
+ /**
+ * Called once a new Searcher is opened.
+ * @param searcher the searcer to warm
+ * @param isTopLevelReader true iff the searcher is build from a top-level reader.
+ * Otherwise the searcher might be build from a leaf reader to warm in isolation
+ */
+ void warm(Engine.Searcher searcher, boolean isTopLevelReader);
+ }
}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index c3015c6e560..f4337de266e 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -61,8 +61,7 @@ public final class EngineConfig {
private final String codecName;
private final ThreadPool threadPool;
private final ShardIndexingService indexingService;
- @Nullable
- private final IndicesWarmer warmer;
+ private final Engine.Warmer warmer;
private final Store store;
private final SnapshotDeletionPolicy deletionPolicy;
private final MergePolicy mergePolicy;
@@ -116,7 +115,7 @@ public final class EngineConfig {
* Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
*/
public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService,
- Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
+ Settings indexSettings, Engine.Warmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer,
Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig, TimeValue flushMergesAfter) {
@@ -124,7 +123,7 @@ public final class EngineConfig {
this.indexSettings = indexSettings;
this.threadPool = threadPool;
this.indexingService = indexingService;
- this.warmer = warmer;
+ this.warmer = warmer == null ? (a,b) -> {} : warmer;
this.store = store;
this.deletionPolicy = deletionPolicy;
this.mergePolicy = mergePolicy;
@@ -267,11 +266,9 @@ public final class EngineConfig {
}
/**
- * Returns an {@link org.elasticsearch.indices.IndicesWarmer} used to warm new searchers before they are used for searching.
- * Note: This method might retrun null
+ * Returns an {@link org.elasticsearch.index.engine.Engine.Warmer} used to warm new searchers before they are used for searching.
*/
- @Nullable
- public IndicesWarmer getWarmer() {
+ public Engine.Warmer getWarmer() {
return warmer;
}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 5d3b9388582..9ab09a245ea 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -79,8 +79,7 @@ public class InternalEngine extends Engine {
private volatile long lastDeleteVersionPruneTimeMSec;
private final ShardIndexingService indexingService;
- @Nullable
- private final IndicesWarmer warmer;
+ private final Engine.Warmer warmer;
private final Translog translog;
private final ElasticsearchConcurrentMergeScheduler mergeScheduler;
@@ -930,8 +929,7 @@ public class InternalEngine extends Engine {
assert isMergedSegment(esLeafReader);
if (warmer != null) {
final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(esLeafReader, null));
- final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher);
- warmer.warmNewReaders(context);
+ warmer.warm(searcher, false);
}
} catch (Throwable t) {
// Don't fail a merge if the warm-up failed
@@ -955,7 +953,7 @@ public class InternalEngine extends Engine {
/** Extended SearcherFactory that warms the segments if needed when acquiring a new searcher */
final static class SearchFactory extends EngineSearcherFactory {
- private final IndicesWarmer warmer;
+ private final Engine.Warmer warmer;
private final ShardId shardId;
private final ESLogger logger;
private final AtomicBoolean isEngineClosed;
@@ -1014,11 +1012,10 @@ public class InternalEngine extends Engine {
}
if (newSearcher != null) {
- IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new Searcher("new_reader_warming", newSearcher));
- warmer.warmNewReaders(context);
+ warmer.warm(new Searcher("new_reader_warming", newSearcher), false);
}
assert searcher.getIndexReader() instanceof ElasticsearchDirectoryReader : "this class needs an ElasticsearchDirectoryReader but got: " + searcher.getIndexReader().getClass();
- warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new Searcher("top_reader_warming", searcher)));
+ warmer.warm(new Searcher("top_reader_warming", searcher), true);
} catch (Throwable e) {
if (isEngineClosed.get() == false) {
logger.warn("failed to prepare/warm", e);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
index b264bfa4bc3..eb0939d9339 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
@@ -25,7 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.XGeoHashUtils;
+import org.apache.lucene.util.GeoHashUtils;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Strings;
@@ -82,7 +82,7 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
public static final boolean ENABLE_LATLON = false;
public static final boolean ENABLE_GEOHASH = false;
public static final boolean ENABLE_GEOHASH_PREFIX = false;
- public static final int GEO_HASH_PRECISION = XGeoHashUtils.PRECISION;
+ public static final int GEO_HASH_PRECISION = GeoHashUtils.PRECISION;
public static final Explicit IGNORE_MALFORMED = new Explicit(false, false);
public static final Explicit COERCE = new Explicit(false, false);
@@ -705,7 +705,7 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper
}
if (fieldType().isGeohashEnabled()) {
if (geohash == null) {
- geohash = XGeoHashUtils.stringEncode(point.lon(), point.lat());
+ geohash = GeoHashUtils.stringEncode(point.lon(), point.lat());
}
addGeohashField(context, geohash);
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java
index 340094af7ce..d0094344b0b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java
@@ -19,16 +19,14 @@
package org.elasticsearch.index.query;
-import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.XContentParser;
+
import java.io.IOException;
public class FuzzyQueryParser implements QueryParser {
- private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("min_similarity");
-
@Override
public String[] names() {
return new String[]{ FuzzyQueryBuilder.NAME };
@@ -68,7 +66,7 @@ public class FuzzyQueryParser implements QueryParser {
value = parser.objectBytes();
} else if ("boost".equals(currentFieldName)) {
boost = parser.floatValue();
- } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZINESS)) {
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
fuzziness = Fuzziness.parse(parser);
} else if ("prefix_length".equals(currentFieldName) || "prefixLength".equals(currentFieldName)) {
prefixLength = parser.intValue();
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
index a385e978487..97d403ff1c9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
@@ -20,7 +20,7 @@
package org.elasticsearch.index.query;
import org.apache.lucene.search.Query;
-import org.apache.lucene.util.XGeoHashUtils;
+import org.apache.lucene.util.GeoHashUtils;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
@@ -131,7 +131,7 @@ public class GeohashCellQuery {
}
public Builder point(double lat, double lon) {
- this.geohash = XGeoHashUtils.stringEncode(lon, lat);
+ this.geohash = GeoHashUtils.stringEncode(lon, lat);
return this;
}
@@ -205,7 +205,7 @@ public class GeohashCellQuery {
Query query;
if (neighbors) {
- query = create(context, geoFieldType, geohash, XGeoHashUtils.addNeighbors(geohash, new ArrayList(8)));
+ query = create(context, geoFieldType, geohash, GeoHashUtils.addNeighbors(geohash, new ArrayList(8)));
} else {
query = create(context, geoFieldType, geohash, null);
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java b/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
index ede4fa98a76..8793de10367 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
@@ -213,29 +212,20 @@ public class IndexQueryParserService extends AbstractIndexComponent {
/**
* Selectively parses a query from a top level query or query_binary json field from the specified source.
*/
- public ParsedQuery parseQuery(BytesReference source) {
+ public ParsedQuery parseTopLevelQuery(BytesReference source) {
XContentParser parser = null;
try {
- parser = XContentHelper.createParser(source);
- ParsedQuery parsedQuery = null;
- for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
- if (token == XContentParser.Token.FIELD_NAME) {
- String fieldName = parser.currentName();
- if ("query".equals(fieldName)) {
- parsedQuery = parse(parser);
- } else if ("query_binary".equals(fieldName) || "queryBinary".equals(fieldName)) {
- byte[] querySource = parser.binaryValue();
- XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource);
- parsedQuery = parse(qSourceParser);
- } else {
- throw new ParsingException(parser.getTokenLocation(), "request does not support [" + fieldName + "]");
- }
- }
+ parser = XContentFactory.xContent(source).createParser(source);
+ QueryShardContext queryShardContext = cache.get();
+ queryShardContext.reset(parser);
+ queryShardContext.parseFieldMatcher(parseFieldMatcher);
+ try {
+ QueryBuilder queryBuilder = queryShardContext.parseContext().parseTopLevelQueryBuilder();
+ Query query = toQuery(queryBuilder, queryShardContext);
+ return new ParsedQuery(query, queryShardContext.copyNamedQueries());
+ } finally {
+ queryShardContext.reset(null);
}
- if (parsedQuery == null) {
- throw new ParsingException(parser.getTokenLocation(), "Required query is missing");
- }
- return parsedQuery;
} catch (ParsingException | QueryShardException e) {
throw e;
} catch (Throwable e) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java
index 674cad70872..416da97942e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryParser.java
@@ -32,8 +32,8 @@ import java.util.Collection;
*/
public class IndicesQueryParser implements QueryParser {
- private static final ParseField QUERY_FIELD = new ParseField("query", "filter");
- private static final ParseField NO_MATCH_QUERY = new ParseField("no_match_query", "no_match_filter");
+ private static final ParseField QUERY_FIELD = new ParseField("query");
+ private static final ParseField NO_MATCH_QUERY = new ParseField("no_match_query");
@Override
public String[] names() {
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
index 1fabfede29d..044a49d23d7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
@@ -20,16 +20,15 @@
package org.elasticsearch.index.query;
import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.support.QueryInnerHits;
+
import java.io.IOException;
public class NestedQueryParser implements QueryParser {
- private static final ParseField FILTER_FIELD = new ParseField("filter").withAllDeprecated("query");
private static final NestedQueryBuilder PROTOTYPE = new NestedQueryBuilder("", EmptyQueryBuilder.PROTOTYPE);
@Override
@@ -54,8 +53,6 @@ public class NestedQueryParser implements QueryParser {
} else if (token == XContentParser.Token.START_OBJECT) {
if ("query".equals(currentFieldName)) {
query = parseContext.parseInnerQueryBuilder();
- } else if (parseContext.parseFieldMatcher().match(currentFieldName, FILTER_FIELD)) {
- query = parseContext.parseInnerQueryBuilder();
} else if ("inner_hits".equals(currentFieldName)) {
queryInnerHits = new QueryInnerHits(parser);
} else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java
deleted file mode 100644
index 4ca9e1598e2..00000000000
--- a/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * A filter that simply wraps a query.
- * @deprecated Useless now that queries and filters are merged: pass the
- * query as a filter directly.
- */
-//TODO: remove when https://github.com/elastic/elasticsearch/issues/13326 is fixed
-@Deprecated
-public class QueryFilterBuilder extends AbstractQueryBuilder {
-
- public static final String NAME = "query";
-
- private final QueryBuilder queryBuilder;
-
- static final QueryFilterBuilder PROTOTYPE = new QueryFilterBuilder(EmptyQueryBuilder.PROTOTYPE);
-
- /**
- * A filter that simply wraps a query.
- *
- * @param queryBuilder The query to wrap as a filter
- */
- public QueryFilterBuilder(QueryBuilder queryBuilder) {
- if (queryBuilder == null) {
- throw new IllegalArgumentException("inner query cannot be null");
- }
- this.queryBuilder = queryBuilder;
- }
-
- /**
- * @return the query builder that is wrapped by this {@link QueryFilterBuilder}
- */
- public QueryBuilder innerQuery() {
- return this.queryBuilder;
- }
-
- @Override
- protected void doXContent(XContentBuilder builder, Params params) throws IOException {
- builder.field(NAME);
- queryBuilder.toXContent(builder, params);
- }
-
- @Override
- protected Query doToQuery(QueryShardContext context) throws IOException {
- // inner query builder can potentially be `null`, in that case we ignore it
- Query innerQuery = this.queryBuilder.toQuery(context);
- if (innerQuery == null) {
- return null;
- }
- return new ConstantScoreQuery(innerQuery);
- }
-
- @Override
- protected void setFinalBoost(Query query) {
- //no-op this query doesn't support boost
- }
-
- @Override
- protected int doHashCode() {
- return Objects.hash(queryBuilder);
- }
-
- @Override
- protected boolean doEquals(QueryFilterBuilder other) {
- return Objects.equals(queryBuilder, other.queryBuilder);
- }
-
- @Override
- protected QueryFilterBuilder doReadFrom(StreamInput in) throws IOException {
- QueryBuilder innerQueryBuilder = in.readQuery();
- return new QueryFilterBuilder(innerQueryBuilder);
- }
-
- @Override
- protected void doWriteTo(StreamOutput out) throws IOException {
- out.writeQuery(queryBuilder);
- }
-
- @Override
- public String getWriteableName() {
- return NAME;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 0885d040179..70a6a18aab2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -65,9 +66,47 @@ public class QueryParseContext {
}
/**
- * @return a new QueryBuilder based on the current state of the parser
+ * Parses a top level query including the query element that wraps it
*/
- public QueryBuilder parseInnerQueryBuilder() throws IOException {
+ public QueryBuilder parseTopLevelQueryBuilder() {
+ try {
+ QueryBuilder queryBuilder = null;
+ for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ String fieldName = parser.currentName();
+ if ("query".equals(fieldName)) {
+ queryBuilder = parseInnerQueryBuilder();
+ } else if ("query_binary".equals(fieldName) || "queryBinary".equals(fieldName)) {
+ byte[] querySource = parser.binaryValue();
+ XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource);
+ QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry);
+ queryParseContext.reset(qSourceParser);
+ try {
+ queryParseContext.parseFieldMatcher(parseFieldMatcher);
+ queryBuilder = queryParseContext.parseInnerQueryBuilder();
+ } finally {
+ queryParseContext.reset(null);
+ }
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "request does not support [" + parser.currentName() + "]");
+ }
+ }
+ }
+ if (queryBuilder == null) {
+ throw new ParsingException(parser.getTokenLocation(), "Required query is missing");
+ }
+ return queryBuilder;
+ } catch (ParsingException e) {
+ throw e;
+ } catch (Throwable e) {
+ throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e);
+ }
+ }
+
+ /**
+ * Parses a query excluding the query element that wraps it
+ */
+ public QueryBuilder parseInnerQueryBuilder() throws IOException {
// move to START object
XContentParser.Token token;
if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
index f5dbb250805..9dfee553b55 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
@@ -19,7 +19,6 @@
package org.elasticsearch.index.query;
-import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.Fuzziness;
@@ -35,8 +34,6 @@ import java.util.Map;
*/
public class QueryStringQueryParser implements QueryParser {
- private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("fuzzy_min_sim");
-
@Override
public String[] names() {
return new String[]{QueryStringQueryBuilder.NAME, Strings.toCamelCase(QueryStringQueryBuilder.NAME)};
@@ -134,7 +131,7 @@ public class QueryStringQueryParser implements QueryParser {
fuzzyRewrite = parser.textOrNull();
} else if ("phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) {
phraseSlop = parser.intValue();
- } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZINESS)) {
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
fuzziness = Fuzziness.parse(parser);
} else if ("boost".equals(currentFieldName)) {
boost = parser.floatValue();
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
index f8b0deaf9be..b38552663d1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
@@ -70,7 +70,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
rethrowUnlessLenient(e);
}
}
- return super.simplify(bq.build());
+ return simplify(bq.build());
}
/**
@@ -93,7 +93,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
rethrowUnlessLenient(e);
}
}
- return super.simplify(bq.build());
+ return simplify(bq.build());
}
@Override
@@ -111,7 +111,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
rethrowUnlessLenient(e);
}
}
- return super.simplify(bq.build());
+ return simplify(bq.build());
}
/**
@@ -140,7 +140,19 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
return rethrowUnlessLenient(e);
}
}
- return super.simplify(bq.build());
+ return simplify(bq.build());
+ }
+
+ /**
+ * Override of lucenes SimpleQueryParser that doesn't simplify for the 1-clause case.
+ */
+ @Override
+ protected Query simplify(BooleanQuery bq) {
+ if (bq.clauses().isEmpty()) {
+ return null;
+ } else {
+ return bq;
+ }
}
/**
@@ -295,7 +307,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
// For further reasoning see
// https://issues.apache.org/jira/browse/LUCENE-4021
return (Objects.equals(locale.toLanguageTag(), other.locale.toLanguageTag())
- && Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms)
+ && Objects.equals(lowercaseExpandedTerms, other.lowercaseExpandedTerms)
&& Objects.equals(lenient, other.lenient)
&& Objects.equals(analyzeWildcard, other.analyzeWildcard));
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
index a93c60ec147..2b221ed9ab0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
@@ -20,6 +20,8 @@
package org.elasticsearch.index.query;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.Strings;
@@ -285,8 +287,20 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder 1
+ && ((booleanQuery.clauses().iterator().next().getQuery() instanceof BooleanQuery) == false)) {
+ // special case for one term query and more than one field: (f1:t1 f2:t1 f3:t1)
+ // we need to wrap this in additional BooleanQuery so minimum_should_match is applied correctly
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ builder.add(new BooleanClause(booleanQuery, Occur.SHOULD));
+ booleanQuery = builder.build();
+ }
+ if (minimumShouldMatch != null) {
+ booleanQuery = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch);
+ }
+ query = booleanQuery;
}
return query;
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 97508a8a16f..c90034cb04d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -57,24 +57,18 @@ public class TermsQueryBuilder extends AbstractQueryBuilder {
static final TermsQueryBuilder PROTOTYPE = new TermsQueryBuilder("field", "value");
- public static final boolean DEFAULT_DISABLE_COORD = false;
-
private final String fieldName;
private final List