diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbb01426fde..cc0d2ba6e35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -141,7 +141,7 @@ Please follow these formatting guidelines: * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them. * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value. - * IntelliJ: `Preferences->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value. + * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value. * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so. To create a distribution from the source, simply run: diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 82a14cfebb9..d1d04ea27b5 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 7.0.0-alpha1 -lucene = 7.1.0 +lucene = 7.2.0-snapshot-8c94404 # optional dependencies spatial4j = 0.6 diff --git a/core/cli/src/main/java/org/elasticsearch/cli/Command.java b/core/cli/src/main/java/org/elasticsearch/cli/Command.java index 78a9f31283d..34ede7ccf94 100644 --- a/core/cli/src/main/java/org/elasticsearch/cli/Command.java +++ b/core/cli/src/main/java/org/elasticsearch/cli/Command.java @@ -38,6 +38,8 @@ public abstract class Command implements Closeable { /** A description of the command, used in the help output. */ protected final String description; + private final Runnable beforeMain; + /** The option parser for this command. */ protected final OptionParser parser = new OptionParser(); @@ -46,8 +48,15 @@ public abstract class Command implements Closeable { private final OptionSpec verboseOption = parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output").availableUnless(silentOption); - public Command(String description) { + /** + * Construct the command with the specified command description and runnable to execute before main is invoked. + * + * @param description the command description + * @param beforeMain the before-main runnable + */ + public Command(final String description, final Runnable beforeMain) { this.description = description; + this.beforeMain = beforeMain; } private Thread shutdownHookThread; @@ -75,7 +84,7 @@ public abstract class Command implements Closeable { Runtime.getRuntime().addShutdownHook(shutdownHookThread); } - beforeExecute(); + beforeMain.run(); try { mainWithoutErrorHandling(args, terminal); @@ -93,12 +102,6 @@ public abstract class Command implements Closeable { return ExitCodes.OK; } - /** - * Setup method to be executed before parsing or execution of the command being run. Any exceptions thrown by the - * method will not be cleanly caught by the parser. - */ - protected void beforeExecute() {} - /** * Executes the command, but all errors are thrown. */ diff --git a/core/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java b/core/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java index 16754cd7bf1..ba6b447792a 100644 --- a/core/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java +++ b/core/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java @@ -35,8 +35,14 @@ public class MultiCommand extends Command { private final NonOptionArgumentSpec arguments = parser.nonOptions("command"); - public MultiCommand(String description) { - super(description); + /** + * Construct the multi-command with the specified command description and runnable to execute before main is invoked. + * + * @param description the multi-command description + * @param beforeMain the before-main runnable + */ + public MultiCommand(final String description, final Runnable beforeMain) { + super(description, beforeMain); parser.posixlyCorrect(true); } diff --git a/core/licenses/lucene-analyzers-common-7.1.0.jar.sha1 b/core/licenses/lucene-analyzers-common-7.1.0.jar.sha1 deleted file mode 100644 index 880d261cb89..00000000000 --- a/core/licenses/lucene-analyzers-common-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a508bf6b580471ee568dab7d2acfedfa5aadce70 \ No newline at end of file diff --git a/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..36a620fef18 --- /dev/null +++ b/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +4c515e5152e6938129a5e97c5afb5b3b360faed3 \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.1.0.jar.sha1 b/core/licenses/lucene-backward-codecs-7.1.0.jar.sha1 deleted file mode 100644 index ec597be207d..00000000000 --- a/core/licenses/lucene-backward-codecs-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -804a7ce82bba3d085733486bfde4846ecb77ce01 \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..5616175485e --- /dev/null +++ b/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +406c6cc0f8c2a47d42a1e343eaf2ad939fee905c \ No newline at end of file diff --git a/core/licenses/lucene-core-7.1.0.jar.sha1 b/core/licenses/lucene-core-7.1.0.jar.sha1 deleted file mode 100644 index 9e811299660..00000000000 --- a/core/licenses/lucene-core-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dd291b7ebf4845483895724d2562214dc7f40049 \ No newline at end of file diff --git a/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..c0abd20b083 --- /dev/null +++ b/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +4c93f7fbc6e0caf87f7948b8481d80e0167133bf \ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.1.0.jar.sha1 b/core/licenses/lucene-grouping-7.1.0.jar.sha1 deleted file mode 100644 index 3c4963f4460..00000000000 --- a/core/licenses/lucene-grouping-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0732d16c16421fca058a2a07ca4081ec7696365b \ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..823ae2c5b48 --- /dev/null +++ b/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +b078ca50c6d579085c7755b4fd8de60711964dcc \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.1.0.jar.sha1 b/core/licenses/lucene-highlighter-7.1.0.jar.sha1 deleted file mode 100644 index 87f841e1467..00000000000 --- a/core/licenses/lucene-highlighter-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -596550daabae765ad685112e0fe7c4f0fdfccb3f \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..9f936b25729 --- /dev/null +++ b/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +fc5e61c8879f22b65ee053f1665bc9f13af79c1d \ No newline at end of file diff --git a/core/licenses/lucene-join-7.1.0.jar.sha1 b/core/licenses/lucene-join-7.1.0.jar.sha1 deleted file mode 100644 index 774ec13c614..00000000000 --- a/core/licenses/lucene-join-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5f26dd64c195258a81175772ef7fe105e7d60a26 \ No newline at end of file diff --git a/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..9bb132317db --- /dev/null +++ b/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +9a10839d3dfe7b369f0af8a78a630ee4d82e678e \ No newline at end of file diff --git a/core/licenses/lucene-memory-7.1.0.jar.sha1 b/core/licenses/lucene-memory-7.1.0.jar.sha1 deleted file mode 100644 index 17264d5e43d..00000000000 --- a/core/licenses/lucene-memory-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3ef64c58d0c09ca40d848efa96b585b7476271f2 \ No newline at end of file diff --git a/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..f7f827c85d1 --- /dev/null +++ b/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +d45f2f51cf6f47a66ecafddecb83c1e08eb4061f \ No newline at end of file diff --git a/core/licenses/lucene-misc-7.1.0.jar.sha1 b/core/licenses/lucene-misc-7.1.0.jar.sha1 deleted file mode 100644 index 6fb92dee458..00000000000 --- a/core/licenses/lucene-misc-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1496ee5fa62206ee5ddf51042a340d6a9ee3b5de \ No newline at end of file diff --git a/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..fde0edc0d99 --- /dev/null +++ b/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +19cb7362be57104ad891259060af80fb4679e92c \ No newline at end of file diff --git a/core/licenses/lucene-queries-7.1.0.jar.sha1 b/core/licenses/lucene-queries-7.1.0.jar.sha1 deleted file mode 100644 index a4028cc2149..00000000000 --- a/core/licenses/lucene-queries-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1554920ab207a3245fa408d022a5c90ad3a1fea3 \ No newline at end of file diff --git a/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..ef6408913d2 --- /dev/null +++ b/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +ae24737048d95f56d0099fea77498324412eef50 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.1.0.jar.sha1 b/core/licenses/lucene-queryparser-7.1.0.jar.sha1 deleted file mode 100644 index 85c745ea911..00000000000 --- a/core/licenses/lucene-queryparser-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5767c15c5ee97926829fd8a4337e434fa95f3c08 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..1670bad8f3f --- /dev/null +++ b/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +a9d3422c9a72106026c19a8f76a4f4e62159ff5c \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.1.0.jar.sha1 b/core/licenses/lucene-sandbox-7.1.0.jar.sha1 deleted file mode 100644 index 4fedc42d2f1..00000000000 --- a/core/licenses/lucene-sandbox-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -691f7b9ac05f3ad2ac7e80733ef70247904bd3ae \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..c8665f0c485 --- /dev/null +++ b/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +66433006587ede3e01899fd6f5e55c8378032c2f \ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.1.0.jar.sha1 b/core/licenses/lucene-spatial-7.1.0.jar.sha1 deleted file mode 100644 index 3cc891f4b4d..00000000000 --- a/core/licenses/lucene-spatial-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c64c04d802badb800516a8a574cb993929c3805 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..7254970749f --- /dev/null +++ b/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +b6b3082ba845f7bd41641b015624f46d4f20afb6 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.1.0.jar.sha1 b/core/licenses/lucene-spatial-extras-7.1.0.jar.sha1 deleted file mode 100644 index 066098d5571..00000000000 --- a/core/licenses/lucene-spatial-extras-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3f1bc1aada8f06b176b782da24b9d7ad9641c41a \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..bb62d7b5997 --- /dev/null +++ b/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +7757cac49cb3e9f1a219346ce95fb80f61f7090e \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.1.0.jar.sha1 b/core/licenses/lucene-spatial3d-7.1.0.jar.sha1 deleted file mode 100644 index 32277c393c9..00000000000 --- a/core/licenses/lucene-spatial3d-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8ded650aed23efb775f17be496e3e3870214e23b \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..78d8c2f5baf --- /dev/null +++ b/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +92991fdcd185883050d9530ccc0d863b7a08e99c \ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.1.0.jar.sha1 b/core/licenses/lucene-suggest-7.1.0.jar.sha1 deleted file mode 100644 index 1d2d0585c63..00000000000 --- a/core/licenses/lucene-suggest-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d0ed1589ebdccf34e888c6efc0134a13a238c85 \ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..5c13208494c --- /dev/null +++ b/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +fb6a94b833a23a17e3721ea2f9679ad770dec48b \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index e882e727429..0fed8316a05 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -93,6 +93,15 @@ public final class MinDocQuery extends Query { final DocIdSetIterator disi = new MinDocIterator(segmentMinDoc, maxDoc); return new ConstantScoreScorer(this, score(), disi); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // Let's not cache this query, the cached iterator would use more memory + // and be slower anyway. + // Also, matches in a given segment depend on the other segments, which + // makes it a bad candidate for per-segment caching. + return false; + } }; } diff --git a/core/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java b/core/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java index b9ed2290350..75fdeee2719 100644 --- a/core/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java @@ -35,9 +35,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.Weight; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Objects; /** @@ -90,6 +88,14 @@ public class SearchAfterSortedDocQuery extends Query { final DocIdSetIterator disi = new MinDocQuery.MinDocIterator(firstDoc, maxDoc); return new ConstantScoreScorer(this, score(), disi); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // If the sort order includes _doc, then the matches in a segment + // may depend on other segments, which makes this query a bad + // candidate for caching + return false; + } }; } diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index c0a723fdb39..372d88c75cd 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -137,7 +137,7 @@ public class Version implements Comparable { public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_7_0_0_alpha1_ID = 7000001; public static final Version V_7_0_0_alpha1 = - new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); + new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_2_0); public static final Version CURRENT = V_7_0_0_alpha1; static { diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index c30dfd360a0..5b20b848f0b 100644 --- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -33,8 +33,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.internal.AliasFilter; @@ -86,6 +88,19 @@ public class TransportExplainAction extends TransportSingleShardAction listener) throws IOException { + IndexService indexService = searchService.getIndicesService().indexServiceSafe(shardId.getIndex()); + IndexShard indexShard = indexService.getShard(shardId.id()); + indexShard.awaitShardSearchActive(b -> { + try { + super.asyncShardOperation(request, shardId, listener); + } catch (Exception ex) { + listener.onFailure(ex); + } + }); + } + @Override protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException { ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId, diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 884af4a3af9..d14db67744d 100644 --- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -19,13 +19,13 @@ package org.elasticsearch.action.get; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.Preference; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -38,6 +38,8 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; + /** * Performs the get operation. */ @@ -76,6 +78,23 @@ public class TransportGetAction extends TransportSingleShardAction listener) throws IOException { + IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + IndexShard indexShard = indexService.getShard(shardId.id()); + if (request.realtime()) { // we are not tied to a refresh cycle here anyway + listener.onResponse(shardOperation(request, shardId)); + } else { + indexShard.awaitShardSearchActive(b -> { + try { + super.asyncShardOperation(request, shardId, listener); + } catch (Exception ex) { + listener.onFailure(ex); + } + }); + } + } + @Override protected GetResponse shardOperation(GetRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 811dcbed3dc..f2b2090dc28 100644 --- a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; @@ -47,6 +48,8 @@ import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.concurrent.Executor; import java.util.function.Supplier; import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException; @@ -78,7 +81,7 @@ public abstract class TransportSingleShardAction listener) throws IOException { + threadPool.executor(this.executor).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + protected void doRun() throws Exception { + listener.onResponse(shardOperation(request, shardId)); + } + }); + } protected abstract Response newResponse(); protected abstract boolean resolveIndex(Request request); @@ -291,11 +307,27 @@ public abstract class TransportSingleShardAction() { + @Override + public void onResponse(Response response) { + try { + channel.sendResponse(response); + } catch (IOException e) { + onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } + } + }); } } - /** * Internal request class that gets built on each node. Holds the original request plus additional info. */ diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 5ff55a6fa55..289f40f1a34 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.termvectors; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; @@ -37,6 +38,8 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; + /** * Performs the get operation. */ @@ -82,6 +85,23 @@ public class TransportTermVectorsAction extends TransportSingleShardAction listener) throws IOException { + IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + IndexShard indexShard = indexService.getShard(shardId.id()); + if (request.realtime()) { // it's a realtime request which is not subject to refresh cycles + listener.onResponse(shardOperation(request, shardId)); + } else { + indexShard.awaitShardSearchActive(b -> { + try { + super.asyncShardOperation(request, shardId, listener); + } catch (Exception ex) { + listener.onFailure(ex); + } + }); + } + } + @Override protected TermVectorsResponse shardOperation(TermVectorsRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java index 54f1528e463..c2e5d2ef11a 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java @@ -38,6 +38,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.AllPermission; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -210,6 +211,7 @@ final class BootstrapChecks { checks.add(new OnOutOfMemoryErrorCheck()); checks.add(new EarlyAccessCheck()); checks.add(new G1GCCheck()); + checks.add(new AllPermissionCheck()); return Collections.unmodifiableList(checks); } @@ -692,4 +694,27 @@ final class BootstrapChecks { } + static class AllPermissionCheck implements BootstrapCheck { + + @Override + public final BootstrapCheckResult check(BootstrapContext context) { + if (isAllPermissionGranted()) { + return BootstrapCheck.BootstrapCheckResult.failure("granting the all permission effectively disables security"); + } + return BootstrapCheckResult.success(); + } + + boolean isAllPermissionGranted() { + final SecurityManager sm = System.getSecurityManager(); + assert sm != null; + try { + sm.checkPermission(new AllPermission()); + } catch (final SecurityException e) { + return false; + } + return true; + } + + } + } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index cfe73459a05..1538f0cdf00 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -51,7 +51,7 @@ class Elasticsearch extends EnvironmentAwareCommand { // visible for testing Elasticsearch() { - super("starts elasticsearch"); + super("starts elasticsearch", () -> {}); // we configure logging later so we override the base class from configuring logging versionOption = parser.acceptsAll(Arrays.asList("V", "version"), "Prints elasticsearch version information and exits"); daemonizeOption = parser.acceptsAll(Arrays.asList("d", "daemonize"), @@ -92,15 +92,6 @@ class Elasticsearch extends EnvironmentAwareCommand { return elasticsearch.main(args, terminal); } - @Override - protected boolean shouldConfigureLoggingWithoutConfig() { - /* - * If we allow logging to be configured without a config before we are ready to read the log4j2.properties file, then we will fail - * to detect uses of logging before it is properly configured. - */ - return false; - } - @Override protected void execute(Terminal terminal, OptionSet options, Environment env) throws UserException { if (options.nonOptionArguments().isEmpty() == false) { diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java index c6692cec08b..6869a6abb71 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java @@ -65,12 +65,10 @@ class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionH } } - // visible for testing static boolean isFatalUncaught(Throwable e) { return e instanceof Error; } - // visible for testing void onFatalUncaught(final String threadName, final Throwable t) { final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get()); logger.error( @@ -78,24 +76,32 @@ class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionH () -> new ParameterizedMessage("fatal error in thread [{}], exiting", threadName), t); } - // visible for testing void onNonFatalUncaught(final String threadName, final Throwable t) { final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get()); logger.warn((org.apache.logging.log4j.util.Supplier) () -> new ParameterizedMessage("uncaught exception in thread [{}]", threadName), t); } - // visible for testing void halt(int status) { - AccessController.doPrivileged(new PrivilegedAction() { - @SuppressForbidden(reason = "halt") - @Override - public Void run() { - // we halt to prevent shutdown hooks from running - Runtime.getRuntime().halt(status); - return null; - } - }); + AccessController.doPrivileged(new PrivilegedHaltAction(status)); + } + + static class PrivilegedHaltAction implements PrivilegedAction { + + private final int status; + + private PrivilegedHaltAction(final int status) { + this.status = status; + } + + @SuppressForbidden(reason = "halt") + @Override + public Void run() { + // we halt to prevent shutdown hooks from running + Runtime.getRuntime().halt(status); + return null; + } + } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index f591780b5ad..3693f5cba58 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -119,7 +119,11 @@ final class Security { Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults)); // enable security manager - final String[] classesThatCanExit = new String[]{ElasticsearchUncaughtExceptionHandler.class.getName(), Command.class.getName()}; + final String[] classesThatCanExit = + new String[]{ + // SecureSM matches class names as regular expressions so we escape the $ that arises from the nested class name + ElasticsearchUncaughtExceptionHandler.PrivilegedHaltAction.class.getName().replace("$", "\\$"), + Command.class.getName()}; System.setSecurityManager(new SecureSM(classesThatCanExit)); // do some basic tests diff --git a/core/src/main/java/org/elasticsearch/cli/CommandLoggingConfigurator.java b/core/src/main/java/org/elasticsearch/cli/CommandLoggingConfigurator.java new file mode 100644 index 00000000000..406c362dd72 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cli/CommandLoggingConfigurator.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cli; + +import org.apache.logging.log4j.Level; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.common.settings.Settings; + +/** + * Holder class for method to configure logging without Elasticsearch configuration files for use in CLI tools that will not read such + * files. + */ +final class CommandLoggingConfigurator { + + /** + * Configures logging without Elasticsearch configuration files based on the system property "es.logger.level" only. As such, any + * logging will be written to the console. + */ + static void configureLoggingWithoutConfig() { + // initialize default for es.logger.level because we will not read the log4j2.properties + final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name()); + final Settings settings = Settings.builder().put("logger.level", loggerLevel).build(); + LogConfigurator.configureWithoutConfig(settings); + } + +} diff --git a/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java index b2bd887e0f6..7d963655957 100644 --- a/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java +++ b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java @@ -22,9 +22,7 @@ package org.elasticsearch.cli; import joptsimple.OptionSet; import joptsimple.OptionSpec; import joptsimple.util.KeyValuePair; -import org.apache.logging.log4j.Level; import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.InternalSettingsPreparer; @@ -40,8 +38,25 @@ public abstract class EnvironmentAwareCommand extends Command { private final OptionSpec settingOption; - public EnvironmentAwareCommand(String description) { - super(description); + /** + * Construct the command with the specified command description. This command will have logging configured without reading Elasticsearch + * configuration files. + * + * @param description the command description + */ + public EnvironmentAwareCommand(final String description) { + this(description, CommandLoggingConfigurator::configureLoggingWithoutConfig); + } + + /** + * Construct the command with the specified command description and runnable to execute before main is invoked. Commands constructed + * with this constructor must take ownership of configuring logging. + * + * @param description the command description + * @param beforeMain the before-main runnable + */ + public EnvironmentAwareCommand(final String description, final Runnable beforeMain) { + super(description, beforeMain); this.settingOption = parser.accepts("E", "Configure a setting").withRequiredArg().ofType(KeyValuePair.class); } @@ -104,26 +119,6 @@ public abstract class EnvironmentAwareCommand extends Command { } } - @Override - protected final void beforeExecute() { - if (shouldConfigureLoggingWithoutConfig()) { - // initialize default for es.logger.level because we will not read the log4j2.properties - final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name()); - final Settings settings = Settings.builder().put("logger.level", loggerLevel).build(); - LogConfigurator.configureWithoutConfig(settings); - } - } - - /** - * Indicate whether or not logging should be configured without reading a log4j2.properties. Most commands should do this because we do - * not configure logging for CLI tools. Only commands that configure logging on their own should not do this. - * - * @return true if logging should be configured without reading a log4j2.properties file - */ - protected boolean shouldConfigureLoggingWithoutConfig() { - return true; - } - /** Execute the command with the initialized {@link Environment}. */ protected abstract void execute(Terminal terminal, OptionSet options, Environment env) throws Exception; diff --git a/core/src/main/java/org/elasticsearch/cli/LoggingAwareCommand.java b/core/src/main/java/org/elasticsearch/cli/LoggingAwareCommand.java new file mode 100644 index 00000000000..94da7f510b1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cli/LoggingAwareCommand.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cli; + +/** + * A command that is aware of logging. This class should be preferred over the base {@link Command} class for any CLI tools that depend on + * core Elasticsearch as they could directly or indirectly touch classes that touch logging and as such logging needs to be configured. + */ +public abstract class LoggingAwareCommand extends Command { + + /** + * Construct the command with the specified command description. This command will have logging configured without reading Elasticsearch + * configuration files. + * + * @param description the command description + */ + public LoggingAwareCommand(final String description) { + super(description, CommandLoggingConfigurator::configureLoggingWithoutConfig); + } + +} diff --git a/core/src/main/java/org/elasticsearch/cli/LoggingAwareMultiCommand.java b/core/src/main/java/org/elasticsearch/cli/LoggingAwareMultiCommand.java new file mode 100644 index 00000000000..e22a4f22e83 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cli/LoggingAwareMultiCommand.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cli; + +/** + * A multi-command that is aware of logging. This class should be preferred over the base {@link MultiCommand} class for any CLI tools that + * depend on core Elasticsearch as they could directly or indirectly touch classes that touch logging and as such logging needs to be + * configured. + */ +public abstract class LoggingAwareMultiCommand extends MultiCommand { + + /** + * Construct the command with the specified command description. This command will have logging configured without reading Elasticsearch + * configuration files. + * + * @param description the command description + */ + public LoggingAwareMultiCommand(final String description) { + super(description, CommandLoggingConfigurator::configureLoggingWithoutConfig); + } + +} diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 597fa970a57..ba1450d1fb8 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -30,16 +30,19 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FieldDoc; @@ -650,6 +653,21 @@ public class Lucene { return LenientParser.parse(toParse, defaultValue); } + /** + * Tries to extract a segment reader from the given index reader. + * If no SegmentReader can be extracted an {@link IllegalStateException} is thrown. + */ + public static SegmentReader segmentReader(LeafReader reader) { + if (reader instanceof SegmentReader) { + return (SegmentReader) reader; + } else if (reader instanceof FilterLeafReader) { + final FilterLeafReader fReader = (FilterLeafReader) reader; + return segmentReader(FilterLeafReader.unwrap(fReader)); + } + // hard fail - we can't get a SegmentReader + throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]"); + } + @SuppressForbidden(reason = "Version#parseLeniently() used in a central place") private static final class LenientParser { public static Version parse(String toParse, Version defaultValue) { @@ -675,10 +693,6 @@ public class Lucene { throw new IllegalStateException(message); } @Override - public int freq() throws IOException { - throw new IllegalStateException(message); - } - @Override public int docID() { throw new IllegalStateException(message); } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index a415b81cddd..f7735dd8197 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -333,6 +333,13 @@ public class FunctionScoreQuery extends Query { } return expl; } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // If minScore is not null, then matches depend on statistics of the + // top-level reader. + return minScore == null; + } } static class FunctionFactorScorer extends FilterScorer { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java index b4b87bda6d8..8e21c1af41a 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java @@ -59,11 +59,6 @@ final class MinScoreScorer extends Scorer { return in.score(); } - @Override - public int freq() throws IOException { - return in.freq(); - } - @Override public DocIdSetIterator iterator() { return TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator()); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java index 2fbc14a1fd1..bcca4c4a035 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java @@ -50,11 +50,6 @@ public class ScriptScoreFunction extends ScoreFunction { return score; } - @Override - public int freq() throws IOException { - throw new UnsupportedOperationException(); - } - @Override public DocIdSetIterator iterator() { throw new UnsupportedOperationException(); diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index b3b0fdb8cf8..63c5e4d5ab5 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.seqno.LocalCheckpointTracker; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.Store; @@ -135,6 +134,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING, IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING, + IndexSettings.INDEX_SEARCH_IDLE_AFTER, IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY, FieldMapper.IGNORE_MALFORMED_SETTING, FieldMapper.COERCE_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java index 16818341cbd..b3d448dae50 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java +++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java @@ -19,13 +19,14 @@ package org.elasticsearch.common.settings; +import org.elasticsearch.cli.LoggingAwareMultiCommand; import org.elasticsearch.cli.MultiCommand; import org.elasticsearch.cli.Terminal; /** * A cli tool for managing secrets in the elasticsearch keystore. */ -public class KeyStoreCli extends MultiCommand { +public class KeyStoreCli extends LoggingAwareMultiCommand { private KeyStoreCli() { super("A tool for managing settings stored in the elasticsearch keystore"); @@ -39,4 +40,5 @@ public class KeyStoreCli extends MultiCommand { public static void main(String[] args) throws Exception { exit(new KeyStoreCli().main(args, Terminal.DEFAULT)); } + } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index d192e8781d6..78489965e39 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.NodeEnvironment; @@ -624,6 +625,27 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust } } if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) { + // once we change the refresh interval we schedule yet another refresh + // to ensure we are in a clean and predictable state. + // it doesn't matter if we move from or to -1 in both cases we want + // docs to become visible immediately. This also flushes all pending indexing / search reqeusts + // that are waiting for a refresh. + threadPool.executor(ThreadPool.Names.REFRESH).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.warn("forced refresh failed after interval change", e); + } + + @Override + protected void doRun() throws Exception { + maybeRefreshEngine(true); + } + + @Override + public boolean isForceExecution() { + return true; + } + }); rescheduleRefreshTasks(); } final Translog.Durability durability = indexSettings.getTranslogDurability(); @@ -686,17 +708,13 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust } } - private void maybeRefreshEngine() { - if (indexSettings.getRefreshInterval().millis() > 0) { + private void maybeRefreshEngine(boolean force) { + if (indexSettings.getRefreshInterval().millis() > 0 || force) { for (IndexShard shard : this.shards.values()) { - if (shard.isReadAllowed()) { - try { - if (shard.isRefreshNeeded()) { - shard.refresh("schedule"); - } - } catch (IndexShardClosedException | AlreadyClosedException ex) { - // fine - continue; - } + try { + shard.scheduledRefresh(); + } catch (IndexShardClosedException | AlreadyClosedException ex) { + // fine - continue; } } } @@ -896,7 +914,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust @Override protected void runInternal() { - indexService.maybeRefreshEngine(); + indexService.maybeRefreshEngine(false); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index 9e390fb5b22..bf498d3d07d 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -62,6 +62,9 @@ public final class IndexSettings { public static final Setting INDEX_TRANSLOG_SYNC_INTERVAL_SETTING = Setting.timeSetting("index.translog.sync_interval", TimeValue.timeValueSeconds(5), TimeValue.timeValueMillis(100), Property.IndexScope); + public static final Setting INDEX_SEARCH_IDLE_AFTER = + Setting.timeSetting("index.search.idle.after", TimeValue.timeValueSeconds(30), + TimeValue.timeValueMinutes(0), Property.IndexScope, Property.Dynamic); public static final Setting INDEX_TRANSLOG_DURABILITY_SETTING = new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(), (value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), Property.Dynamic, Property.IndexScope); @@ -262,6 +265,8 @@ public final class IndexSettings { private volatile int maxNgramDiff; private volatile int maxShingleDiff; private volatile boolean TTLPurgeDisabled; + private volatile TimeValue searchIdleAfter; + /** * The maximum number of refresh listeners allows on this shard. */ @@ -371,6 +376,7 @@ public final class IndexSettings { maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL); this.mergePolicyConfig = new MergePolicyConfig(logger, this); this.indexSortConfig = new IndexSortConfig(this); + searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); singleType = INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexMetaData.getSettings()); // get this from metadata - it's not registered if ((singleType || version.before(Version.V_6_0_0_alpha1)) == false) { throw new AssertionError(index.toString() + "multiple types are only allowed on pre 6.x indices but version is: [" @@ -411,8 +417,11 @@ public final class IndexSettings { scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners); scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll); scopedSettings.addSettingsUpdateConsumer(DEFAULT_FIELD_SETTING, this::setDefaultFields); + scopedSettings.addSettingsUpdateConsumer(INDEX_SEARCH_IDLE_AFTER, this::setSearchIdleAfter); } + private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; } + private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) { this.flushThresholdSize = byteSizeValue; } @@ -752,4 +761,16 @@ public final class IndexSettings { } public IndexScopedSettings getScopedSettings() { return scopedSettings;} + + /** + * Returns true iff the refresh setting exists or in other words is explicitly set. + */ + public boolean isExplicitRefresh() { + return INDEX_REFRESH_INTERVAL_SETTING.exists(settings); + } + + /** + * Returns the time that an index shard becomes search idle unless it's accessed in between + */ + public TimeValue getSearchIdleAfter() { return searchIdleAfter; } } diff --git a/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java index b07b68d82b8..add4a443903 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java +++ b/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java @@ -71,18 +71,12 @@ class CombinedDeletionPolicy extends IndexDeletionPolicy { } private void setLastCommittedTranslogGeneration(List commits) throws IOException { - // We need to keep translog since the smallest translog generation of un-deleted commits. - // However, there are commits that are not deleted just because they are being snapshotted (rather than being kept by the policy). - // TODO: We need to distinguish those commits and skip them in calculating the minimum required translog generation. - long minRequiredGen = Long.MAX_VALUE; - for (IndexCommit indexCommit : commits) { - if (indexCommit.isDeleted() == false) { - long translogGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); - minRequiredGen = Math.min(translogGen, minRequiredGen); - } - } - assert minRequiredGen != Long.MAX_VALUE : "All commits are deleted"; - translogDeletionPolicy.setMinTranslogGenerationForRecovery(minRequiredGen); + // when opening an existing lucene index, we currently always open the last commit. + // we therefore use the translog gen as the one that will be required for recovery + final IndexCommit indexCommit = commits.get(commits.size() - 1); + assert indexCommit.isDeleted() == false : "last commit is deleted"; + long minGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); + translogDeletionPolicy.setMinTranslogGenerationForRecovery(minGen); } public SnapshotDeletionPolicy getIndexDeletionPolicy() { diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index 99410d9f624..27575a9c354 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexReader; @@ -79,10 +78,12 @@ import java.util.Arrays; import java.util.Base64; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -91,6 +92,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiFunction; +import java.util.stream.Collectors; public abstract class Engine implements Closeable { @@ -143,27 +145,12 @@ public abstract class Engine implements Closeable { return a.ramBytesUsed(); } - /** - * Tries to extract a segment reader from the given index reader. - * If no SegmentReader can be extracted an {@link IllegalStateException} is thrown. - */ - protected static SegmentReader segmentReader(LeafReader reader) { - if (reader instanceof SegmentReader) { - return (SegmentReader) reader; - } else if (reader instanceof FilterLeafReader) { - final FilterLeafReader fReader = (FilterLeafReader) reader; - return segmentReader(FilterLeafReader.unwrap(fReader)); - } - // hard fail - we can't get a SegmentReader - throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]"); - } - /** * Returns whether a leaf reader comes from a merge (versus flush or addIndexes). */ protected static boolean isMergedSegment(LeafReader reader) { // We expect leaves to be segment readers - final Map diagnostics = segmentReader(reader).getSegmentInfo().info.getDiagnostics(); + final Map diagnostics = Lucene.segmentReader(reader).getSegmentInfo().info.getDiagnostics(); final String source = diagnostics.get(IndexWriter.SOURCE); assert Arrays.asList(IndexWriter.SOURCE_ADDINDEXES_READERS, IndexWriter.SOURCE_FLUSH, IndexWriter.SOURCE_MERGE).contains(source) : "Unknown source " + source; @@ -608,25 +595,40 @@ public abstract class Engine implements Closeable { */ public final SegmentsStats segmentsStats(boolean includeSegmentFileSizes) { ensureOpen(); - try (Searcher searcher = acquireSearcher("segments_stats")) { - SegmentsStats stats = new SegmentsStats(); - for (LeafReaderContext reader : searcher.reader().leaves()) { - final SegmentReader segmentReader = segmentReader(reader.reader()); - stats.add(1, segmentReader.ramBytesUsed()); - stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); - stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); - stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); - stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); - stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); - stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); + Set segmentName = new HashSet<>(); + SegmentsStats stats = new SegmentsStats(); + try (Searcher searcher = acquireSearcher("segments_stats", SearcherScope.INTERNAL)) { + for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + fillSegmentStats(segmentReader, includeSegmentFileSizes, stats); + segmentName.add(segmentReader.getSegmentName()); + } + } - if (includeSegmentFileSizes) { - // TODO: consider moving this to StoreStats - stats.addFileSizes(getSegmentFileSizes(segmentReader)); + try (Searcher searcher = acquireSearcher("segments_stats", SearcherScope.EXTERNAL)) { + for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + if (segmentName.contains(segmentReader.getSegmentName()) == false) { + fillSegmentStats(segmentReader, includeSegmentFileSizes, stats); } } - writerSegmentStats(stats); - return stats; + } + writerSegmentStats(stats); + return stats; + } + + private void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { + stats.add(1, segmentReader.ramBytesUsed()); + stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); + stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); + stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); + stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); + stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); + stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); + + if (includeSegmentFileSizes) { + // TODO: consider moving this to StoreStats + stats.addFileSizes(getSegmentFileSizes(segmentReader)); } } @@ -716,30 +718,18 @@ public abstract class Engine implements Closeable { ensureOpen(); Map segments = new HashMap<>(); // first, go over and compute the search ones... - try (Searcher searcher = acquireSearcher("segments")){ - for (LeafReaderContext reader : searcher.reader().leaves()) { - final SegmentReader segmentReader = segmentReader(reader.reader()); - SegmentCommitInfo info = segmentReader.getSegmentInfo(); - assert !segments.containsKey(info.info.name); - Segment segment = new Segment(info.info.name); - segment.search = true; - segment.docCount = reader.reader().numDocs(); - segment.delDocCount = reader.reader().numDeletedDocs(); - segment.version = info.info.getVersion(); - segment.compound = info.info.getUseCompoundFile(); - try { - segment.sizeInBytes = info.sizeInBytes(); - } catch (IOException e) { - logger.trace((Supplier) () -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); + try (Searcher searcher = acquireSearcher("segments", SearcherScope.EXTERNAL)){ + for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { + fillSegmentInfo(Lucene.segmentReader(ctx.reader()), verbose, true, segments); + } + } + + try (Searcher searcher = acquireSearcher("segments", SearcherScope.INTERNAL)){ + for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + if (segments.containsKey(segmentReader.getSegmentName()) == false) { + fillSegmentInfo(segmentReader, verbose, false, segments); } - segment.memoryInBytes = segmentReader.ramBytesUsed(); - segment.segmentSort = info.info.getIndexSort(); - if (verbose) { - segment.ramTree = Accountables.namedAccountable("root", segmentReader); - } - segment.attributes = info.info.getAttributes(); - // TODO: add more fine grained mem stats values to per segment info here - segments.put(info.info.name, segment); } } @@ -769,16 +759,34 @@ public abstract class Engine implements Closeable { } Segment[] segmentsArr = segments.values().toArray(new Segment[segments.values().size()]); - Arrays.sort(segmentsArr, new Comparator() { - @Override - public int compare(Segment o1, Segment o2) { - return (int) (o1.getGeneration() - o2.getGeneration()); - } - }); - + Arrays.sort(segmentsArr, Comparator.comparingLong(Segment::getGeneration)); return segmentsArr; } + private void fillSegmentInfo(SegmentReader segmentReader, boolean verbose, boolean search, Map segments) { + SegmentCommitInfo info = segmentReader.getSegmentInfo(); + assert segments.containsKey(info.info.name) == false; + Segment segment = new Segment(info.info.name); + segment.search = search; + segment.docCount = segmentReader.numDocs(); + segment.delDocCount = segmentReader.numDeletedDocs(); + segment.version = info.info.getVersion(); + segment.compound = info.info.getUseCompoundFile(); + try { + segment.sizeInBytes = info.sizeInBytes(); + } catch (IOException e) { + logger.trace((Supplier) () -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); + } + segment.memoryInBytes = segmentReader.ramBytesUsed(); + segment.segmentSort = info.info.getIndexSort(); + if (verbose) { + segment.ramTree = Accountables.namedAccountable("root", segmentReader); + } + segment.attributes = info.info.getAttributes(); + // TODO: add more fine grained mem stats values to per segment info here + segments.put(info.info.name, segment); + } + /** * The list of segments in the engine. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java index 3b6b206c212..2bf2abac957 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java @@ -29,7 +29,7 @@ import java.io.IOException; /** * A {@link SortedSetDocValues} implementation that returns ordinals that are global. */ -public class GlobalOrdinalMapping extends SortedSetDocValues { +final class GlobalOrdinalMapping extends SortedSetDocValues { private final SortedSetDocValues values; private final OrdinalMap ordinalMap; @@ -49,7 +49,7 @@ public class GlobalOrdinalMapping extends SortedSetDocValues { return ordinalMap.getValueCount(); } - public final long getGlobalOrd(long segmentOrd) { + public long getGlobalOrd(long segmentOrd) { return mapping.get(segmentOrd); } diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 91d5bc9a027..6e0b8159ded 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -153,23 +153,17 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override public boolean equals(Object obj) { - // TODO: Do this if/when we can assume scripts are pure functions - // and they have a reliable equals impl - /*if (this == obj) - return true; if (sameClassAs(obj) == false) return false; ScriptQuery other = (ScriptQuery) obj; - return Objects.equals(script, other.script);*/ - return this == obj; + return Objects.equals(script, other.script); } @Override public int hashCode() { - // TODO: Do this if/when we can assume scripts are pure functions - // and they have a reliable equals impl - // return Objects.hash(classHash(), script); - return System.identityHashCode(this); + int h = classHash(); + h = 31 * h + script.hashCode(); + return h; } @Override @@ -196,6 +190,14 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder }; return new ConstantScoreScorer(this, score(), twoPhase); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // TODO: Change this to true when we can assume that scripts are pure functions + // ie. the return value is always the same given the same conditions and may not + // depend on the current timestamp, other documents, etc. + return false; + } }; } } diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index 0947a67212d..b704c89819f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DoubleValues; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; import org.apache.lucene.search.Query; @@ -290,13 +291,18 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder pendingRefreshLocation = new AtomicReference<>(); + public IndexShard( ShardRouting shardRouting, IndexSettings indexSettings, @@ -298,6 +305,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl searcherWrapper = indexSearcherWrapper; primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); refreshListeners = buildRefreshListeners(); + lastSearcherAccess.set(threadPool.relativeTimeInMillis()); persistMetadata(path, indexSettings, shardRouting, null, logger); } @@ -856,15 +864,30 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } public DocsStats docStats() { + // we calculate the doc stats based on the internal reader that is more up-to-date and not subject + // to external refreshes. For instance we don't refresh an external reader if we flush and indices with + // index.refresh_interval=-1 won't see any doc stats updates at all. This change will give more accurate statistics + // when indexing but not refreshing in general. Yet, if a refresh happens the internal reader is refresh as well so we are + // safe here. long numDocs = 0; long numDeletedDocs = 0; long sizeInBytes = 0; - List segments = segments(false); - for (Segment segment : segments) { - if (segment.search) { - numDocs += segment.getNumDocs(); - numDeletedDocs += segment.getDeletedDocs(); - sizeInBytes += segment.getSizeInBytes(); + try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) { + // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause + // the next scheduled refresh to go through and refresh the stats as well + markSearcherAccessed(); + for (LeafReaderContext reader : searcher.reader().leaves()) { + // we go on the segment level here to get accurate numbers + final SegmentReader segmentReader = Lucene.segmentReader(reader.reader()); + SegmentCommitInfo info = segmentReader.getSegmentInfo(); + numDocs += reader.reader().numDocs(); + numDeletedDocs += reader.reader().numDeletedDocs(); + try { + sizeInBytes += info.sizeInBytes(); + } catch (IOException e) { + logger.trace((org.apache.logging.log4j.util.Supplier) + () -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); + } } } return new DocsStats(numDocs, numDeletedDocs, sizeInBytes); @@ -949,6 +972,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl public CompletionStats completionStats(String... fields) { CompletionStats completionStats = new CompletionStats(); try (Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) { + // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause + // the next scheduled refresh to go through and refresh the stats as well + markSearcherAccessed(); completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields)); } return completionStats; @@ -1118,6 +1144,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl return acquireSearcher(source, Engine.SearcherScope.EXTERNAL); } + private void markSearcherAccessed() { + lastSearcherAccess.lazySet(threadPool.relativeTimeInMillis()); + } + private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scope) { readAllowed(); final Engine engine = getEngine(); @@ -2393,7 +2423,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl indexSettings::getMaxRefreshListeners, () -> refresh("too_many_listeners"), threadPool.executor(ThreadPool.Names.LISTENER)::execute, - logger); + logger, threadPool.getThreadContext()); } /** @@ -2419,14 +2449,74 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } /** - * Returns true iff one or more changes to the engine are not visible to via the current searcher *or* there are pending - * refresh listeners. - * Otherwise false. + * Executes a scheduled refresh if necessary. * - * @throws AlreadyClosedException if the engine or internal indexwriter in the engine is already closed + * @return true iff the engine got refreshed otherwise false */ - public boolean isRefreshNeeded() { - return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded()); + public boolean scheduledRefresh() { + boolean listenerNeedsRefresh = refreshListeners.refreshNeeded(); + if (isReadAllowed() && (listenerNeedsRefresh || getEngine().refreshNeeded())) { + if (listenerNeedsRefresh == false // if we have a listener that is waiting for a refresh we need to force it + && isSearchIdle() && indexSettings.isExplicitRefresh() == false) { + // lets skip this refresh since we are search idle and + // don't necessarily need to refresh. the next searcher access will register a refreshListener and that will + // cause the next schedule to refresh. + setRefreshPending(); + return false; + } else { + refresh("schedule"); + return true; + } + } + return false; + } + + /** + * Returns true if this shards is search idle + */ + final boolean isSearchIdle() { + return (threadPool.relativeTimeInMillis() - lastSearcherAccess.get()) >= indexSettings.getSearchIdleAfter().getMillis(); + } + + /** + * Returns the last timestamp the searcher was accessed. This is a relative timestamp in milliseconds. + */ + final long getLastSearcherAccess() { + return lastSearcherAccess.get(); + } + + private void setRefreshPending() { + Engine engine = getEngine(); + Translog.Location lastWriteLocation = engine.getTranslog().getLastWriteLocation(); + Translog.Location location; + do { + location = this.pendingRefreshLocation.get(); + if (location != null && lastWriteLocation.compareTo(location) <= 0) { + break; + } + } while (pendingRefreshLocation.compareAndSet(location, lastWriteLocation) == false); + } + + /** + * Registers the given listener and invokes it once the shard is active again and all + * pending refresh translog location has been refreshed. If there is no pending refresh location registered the listener will be + * invoked immediately. + * @param listener the listener to invoke once the pending refresh location is visible. The listener will be called with + * true if the listener was registered to wait for a refresh. + */ + public final void awaitShardSearchActive(Consumer listener) { + if (isSearchIdle()) { + markSearcherAccessed(); // move the shard into non-search idle + } + final Translog.Location location = pendingRefreshLocation.get(); + if (location != null) { + addRefreshListener(location, (b) -> { + pendingRefreshLocation.compareAndSet(location, null); + listener.accept(true); + }); + } else { + listener.accept(false); + } } /** diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java index f0df6e12b8c..17e824eb046 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java +++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.shard; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.ReferenceManager; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.translog.Translog; import java.io.Closeable; @@ -45,6 +46,7 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, private final Runnable forceRefresh; private final Executor listenerExecutor; private final Logger logger; + private final ThreadContext threadContext; /** * Is this closed? If true then we won't add more listeners and have flushed all pending listeners. @@ -63,11 +65,13 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, */ private volatile Translog.Location lastRefreshedLocation; - public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger) { + public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger, + ThreadContext threadContext) { this.getMaxRefreshListeners = getMaxRefreshListeners; this.forceRefresh = forceRefresh; this.listenerExecutor = listenerExecutor; this.logger = logger; + this.threadContext = threadContext; } /** @@ -98,8 +102,15 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, refreshListeners = listeners; } if (listeners.size() < getMaxRefreshListeners.getAsInt()) { + ThreadContext.StoredContext storedContext = threadContext.newStoredContext(true); + Consumer contextPreservingListener = forced -> { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + storedContext.restore(); + listener.accept(forced); + } + }; // We have a free slot so register the listener - listeners.add(new Tuple<>(location, listener)); + listeners.add(new Tuple<>(location, contextPreservingListener)); return false; } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index 0919630abcc..633894f3c62 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -157,6 +157,13 @@ final class ShardSplittingQuery extends Query { return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length())); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // This is not a regular query, let's not cache it. It wouldn't help + // anyway. + return false; + } }; } diff --git a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java index 668633e07ef..cc9dbdeb63f 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java +++ b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java @@ -19,6 +19,12 @@ package org.elasticsearch.index.translog; +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongObjectHashMap; +import com.carrotsearch.hppc.LongSet; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.index.seqno.SequenceNumbers; + import java.io.Closeable; import java.io.IOException; import java.util.Arrays; @@ -30,32 +36,44 @@ final class MultiSnapshot implements Translog.Snapshot { private final TranslogSnapshot[] translogs; private final int totalOperations; + private int overriddenOperations; private final Closeable onClose; private int index; + private final SeqNoSet seenSeqNo; /** * Creates a new point in time snapshot of the given snapshots. Those snapshots are always iterated in-order. */ MultiSnapshot(TranslogSnapshot[] translogs, Closeable onClose) { this.translogs = translogs; - totalOperations = Arrays.stream(translogs).mapToInt(TranslogSnapshot::totalOperations).sum(); + this.totalOperations = Arrays.stream(translogs).mapToInt(TranslogSnapshot::totalOperations).sum(); + this.overriddenOperations = 0; this.onClose = onClose; - index = 0; + this.seenSeqNo = new SeqNoSet(); + this.index = translogs.length - 1; } - @Override public int totalOperations() { return totalOperations; } + @Override + public int overriddenOperations() { + return overriddenOperations; + } + @Override public Translog.Operation next() throws IOException { - for (; index < translogs.length; index++) { + for (; index >= 0; index--) { final TranslogSnapshot current = translogs[index]; - Translog.Operation op = current.next(); - if (op != null) { // if we are null we move to the next snapshot - return op; + Translog.Operation op; + while ((op = current.next()) != null) { + if (op.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO || seenSeqNo.getAndSet(op.seqNo()) == false) { + return op; + } else { + overriddenOperations++; + } } } return null; @@ -65,4 +83,76 @@ final class MultiSnapshot implements Translog.Snapshot { public void close() throws IOException { onClose.close(); } + + /** + * A wrapper of {@link FixedBitSet} but allows to check if all bits are set in O(1). + */ + private static final class CountedBitSet { + private short onBits; + private final FixedBitSet bitset; + + CountedBitSet(short numBits) { + assert numBits > 0; + this.onBits = 0; + this.bitset = new FixedBitSet(numBits); + } + + boolean getAndSet(int index) { + assert index >= 0; + boolean wasOn = bitset.getAndSet(index); + if (wasOn == false) { + onBits++; + } + return wasOn; + } + + boolean hasAllBitsOn() { + return onBits == bitset.length(); + } + } + + /** + * Sequence numbers from translog are likely to form contiguous ranges, + * thus collapsing a completed bitset into a single entry will reduce memory usage. + */ + static final class SeqNoSet { + static final short BIT_SET_SIZE = 1024; + private final LongSet completedSets = new LongHashSet(); + private final LongObjectHashMap ongoingSets = new LongObjectHashMap<>(); + + /** + * Marks this sequence number and returns true if it is seen before. + */ + boolean getAndSet(long value) { + assert value >= 0; + final long key = value / BIT_SET_SIZE; + + if (completedSets.contains(key)) { + return true; + } + + CountedBitSet bitset = ongoingSets.get(key); + if (bitset == null) { + bitset = new CountedBitSet(BIT_SET_SIZE); + ongoingSets.put(key, bitset); + } + + final boolean wasOn = bitset.getAndSet(Math.toIntExact(value % BIT_SET_SIZE)); + if (bitset.hasAllBitsOn()) { + ongoingSets.remove(key); + completedSets.add(key); + } + return wasOn; + } + + // For testing + long completeSetsSize() { + return completedSets.size(); + } + + // For testing + long ongoingSetsSize() { + return ongoingSets.size(); + } + } } diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index 4373c8d0539..80033833899 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -831,10 +831,19 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC public interface Snapshot extends Closeable { /** - * The total number of operations in the translog. + * The total estimated number of operations in the snapshot. */ int totalOperations(); + /** + * The number of operations have been overridden (eg. superseded) in the snapshot so far. + * If two operations have the same sequence number, the operation with a lower term will be overridden by the operation + * with a higher term. Unlike {@link #totalOperations()}, this value is updated each time after {@link #next()}) is called. + */ + default int overriddenOperations() { + return 0; + } + /** * Returns the next operation in the snapshot or null if we reached the end. */ diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java index 944296d6813..b9cbf032951 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java +++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java @@ -19,13 +19,14 @@ package org.elasticsearch.index.translog; +import org.elasticsearch.cli.LoggingAwareMultiCommand; import org.elasticsearch.cli.MultiCommand; import org.elasticsearch.cli.Terminal; /** * Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool */ -public class TranslogToolCli extends MultiCommand { +public class TranslogToolCli extends LoggingAwareMultiCommand { private TranslogToolCli() { super("A CLI tool for various Elasticsearch translog actions"); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 126ed6d92e9..60741c87f21 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -157,6 +157,11 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, shardKeyMap.add(context.reader()); return in.bulkScorer(context); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return in.isCacheable(ctx); + } } /** Clear all entries that belong to the given index. */ diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 5f692d8e8f5..71ad21c14d7 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -66,6 +66,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.function.Supplier; @@ -567,8 +568,9 @@ public class RecoverySourceHandler { cancellableThreads.executeIO(sendBatch); } - assert expectedTotalOps == skippedOps + totalSentOps - : "expected total [" + expectedTotalOps + "], skipped [" + skippedOps + "], total sent [" + totalSentOps + "]"; + assert expectedTotalOps == snapshot.overriddenOperations() + skippedOps + totalSentOps + : String.format(Locale.ROOT, "expected total [%d], overridden [%d], skipped [%d], total sent [%d]", + expectedTotalOps, snapshot.overriddenOperations(), skippedOps, totalSentOps); logger.trace("sent final batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), expectedTotalOps); diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 84d3d743f64..440867577ea 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -616,7 +616,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); // EMPTY is safe here because RepositoryData#fromXContent calls namedObject - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) { repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen); } catch (NotXContentException e) { logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length()); @@ -628,7 +628,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) { repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser); } } catch (NoSuchFileException e) { diff --git a/core/src/main/java/org/elasticsearch/search/SearchHit.java b/core/src/main/java/org/elasticsearch/search/SearchHit.java index 7566d5ad279..8d434b8af20 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/core/src/main/java/org/elasticsearch/search/SearchHit.java @@ -328,6 +328,14 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable listener) { + ActionListener actionListener = ActionListener.wrap(r -> + threadPool.executor(Names.SEARCH).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + protected void doRun() throws Exception { + listener.onResponse(request); + } + }), listener::onFailure); + IndexShard shardOrNull = indicesService.getShardOrNull(request.shardId()); + if (shardOrNull != null) { + // now we need to check if there is a pending refresh and register + ActionListener finalListener = actionListener; + actionListener = ActionListener.wrap(r -> + shardOrNull.awaitShardSearchActive(b -> finalListener.onResponse(r)), finalListener::onFailure); + } // we also do rewrite on the coordinating node (TransportSearchService) but we also need to do it here for BWC as well as // AliasFilters that might need to be rewritten. These are edge-cases but we are every efficient doing the rewrite here so it's not // adding a lot of overhead - Rewriteable.rewriteAndFetch(request.getRewriteable(), indicesService.getRewriteContext(request::nowInMillis), - ActionListener.wrap(r -> - threadPool.executor(Names.SEARCH).execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } + Rewriteable.rewriteAndFetch(request.getRewriteable(), indicesService.getRewriteContext(request::nowInMillis), actionListener); + - @Override - protected void doRun() throws Exception { - listener.onResponse(request); - } - }), listener::onFailure)); } /** @@ -1003,4 +1013,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis) { return indicesService.getRewriteContext(nowInMillis); } + + public IndicesService getIndicesService() { + return indicesService; + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSource.java index 045d8fa5ed3..db583d14ffd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSource.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.sort.SortOrder; @@ -179,16 +178,10 @@ abstract class CompositeValuesSource>> 1; - BytesRef midVal = mapping.lookupOrd(mid); - int cmp = midVal.compareTo(key); - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; - } - } - return low-1; - } } /** diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java index acdb3f18e70..05d9402230d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java @@ -259,11 +259,6 @@ public class BestDocsDeferringCollector extends DeferringBucketCollector impleme return currentScore; } - @Override - public int freq() throws IOException { - throw new ElasticsearchException("This caching scorer implementation only implements score() and docID()"); - } - @Override public int docID() { return currentDocId; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index a9d8841dd4d..55023eb263f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; -import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -50,6 +49,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.LongUnaryOperator; import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS; @@ -295,9 +295,8 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr */ static class LowCardinality extends GlobalOrdinalsStringTermsAggregator { + private LongUnaryOperator mapping; private IntArray segmentDocCounts; - private SortedSetDocValues globalOrds; - private SortedSetDocValues segmentOrds; LowCardinality(String name, AggregatorFactories factories, @@ -321,14 +320,14 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr @Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { - if (segmentOrds != null) { - mapSegmentCountsToGlobalCounts(); + if (mapping != null) { + mapSegmentCountsToGlobalCounts(mapping); } - globalOrds = valuesSource.globalOrdinalsValues(ctx); - segmentOrds = valuesSource.ordinalsValues(ctx); + final SortedSetDocValues segmentOrds = valuesSource.ordinalsValues(ctx); segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR; final SortedDocValues singleValues = DocValues.unwrapSingleton(segmentOrds); + mapping = valuesSource.globalOrdinalsMapping(ctx); if (singleValues != null) { return new LeafBucketCollectorBase(sub, segmentOrds) { @Override @@ -356,9 +355,10 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr } @Override - protected void doPostCollection() { - if (segmentOrds != null) { - mapSegmentCountsToGlobalCounts(); + protected void doPostCollection() throws IOException { + if (mapping != null) { + mapSegmentCountsToGlobalCounts(mapping); + mapping = null; } } @@ -367,16 +367,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr Releasables.close(segmentDocCounts); } - private void mapSegmentCountsToGlobalCounts() { - // There is no public method in Ordinals.Docs that allows for this mapping... - // This is the cleanest way I can think of so far - - GlobalOrdinalMapping mapping; - if (globalOrds.getValueCount() == segmentOrds.getValueCount()) { - mapping = null; - } else { - mapping = (GlobalOrdinalMapping) globalOrds; - } + private void mapSegmentCountsToGlobalCounts(LongUnaryOperator mapping) throws IOException { for (long i = 1; i < segmentDocCounts.size(); i++) { // We use set(...) here, because we need to reset the slow to 0. // segmentDocCounts get reused over the segments and otherwise counts would be too high. @@ -385,7 +376,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr continue; } final long ord = i - 1; // remember we do +1 when counting - final long globalOrd = mapping == null ? ord : mapping.getGlobalOrd(ord); + final long globalOrd = mapping.applyAsLong(ord); long bucketOrd = getBucketOrd(globalOrd); incrementBucketDocCount(bucketOrd, inc); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java index 386a7da3e64..1027785c577 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java @@ -49,6 +49,8 @@ import java.util.Map; public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory { private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TermsAggregatorFactory.class)); + static Boolean REMAP_GLOBAL_ORDS, COLLECT_SEGMENT_ORDS; + private final BucketOrder order; private final IncludeExclude includeExclude; private final String executionHint; @@ -257,11 +259,13 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory= 0) { + // the missing value exists in the segment, nothing to do + return segmentToGlobalOrd; + } else if (missingGlobalOrd >= 0) { + // the missing value exists in another segment, but not the current one + final long insertedSegmentOrd = -1L - missingSegmentOrd; + final long insertedGlobalOrd = missingGlobalOrd; + return segmentOrd -> { + if (insertedSegmentOrd == segmentOrd) { + return insertedGlobalOrd; + } else if (insertedSegmentOrd > segmentOrd) { + return segmentToGlobalOrd.applyAsLong(segmentOrd); + } else { + return segmentToGlobalOrd.applyAsLong(segmentOrd - 1); + } + }; + } else { + // the missing value exists neither in this segment nor in another segment + final long insertedSegmentOrd = -1L - missingSegmentOrd; + final long insertedGlobalOrd = -1L - missingGlobalOrd; + return segmentOrd -> { + if (insertedSegmentOrd == segmentOrd) { + return insertedGlobalOrd; + } else if (insertedSegmentOrd > segmentOrd) { + return segmentToGlobalOrd.applyAsLong(segmentOrd); + } else { + return 1 + segmentToGlobalOrd.applyAsLong(segmentOrd - 1); + } + }; + } + } + public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, final GeoPoint missing) { return new ValuesSource.GeoPoint() { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 5a69be8108a..b5a109e89cb 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; @@ -47,6 +48,7 @@ import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues; import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import java.io.IOException; +import java.util.function.LongUnaryOperator; public abstract class ValuesSource { @@ -90,6 +92,11 @@ public abstract class ValuesSource { return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } + @Override + public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException { + return LongUnaryOperator.identity(); + } + }; @Override @@ -105,6 +112,10 @@ public abstract class ValuesSource { public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) throws IOException; + /** Returns a mapping from segment ordinals to global ordinals. */ + public abstract LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) + throws IOException; + public long globalMaxOrd(IndexSearcher indexSearcher) throws IOException { IndexReader indexReader = indexSearcher.getIndexReader(); if (indexReader.leaves().isEmpty()) { @@ -142,6 +153,18 @@ public abstract class ValuesSource { final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); } + + @Override + public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException { + final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); + final OrdinalMap map = global.getOrdinalMap(); + if (map == null) { + // segments and global ordinals are the same + return LongUnaryOperator.identity(); + } + final org.apache.lucene.util.LongValues segmentToGlobalOrd = map.getGlobalOrds(context.ord); + return segmentToGlobalOrd::get; + } } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java index da593d57b77..403bf833878 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java @@ -20,21 +20,16 @@ package org.elasticsearch.search.fetch.subphase; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; - public final class FetchSourceSubPhase implements FetchSubPhase { @Override @@ -65,7 +60,17 @@ public final class FetchSourceSubPhase implements FetchSubPhase { final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length()); BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput); - builder.value(value); + if (value != null) { + builder.value(value); + } else { + // This happens if the source filtering could not find the specified in the _source. + // Just doing `builder.value(null)` is valid, but the xcontent validation can't detect what format + // it is. In certain cases, for example response serialization we fail if no xcontent type can't be + // detected. So instead we just return an empty top level object. Also this is in inline with what was + // being return in this situation in 5.x and earlier. + builder.startObject(); + builder.endObject(); + } hitContext.hit().sourceRef(builder.bytes()); } catch (IOException e) { throw new ElasticsearchException("Error filtering source", e); diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 815486d84a7..fce1e323fa7 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -170,6 +170,11 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { throw new UnsupportedOperationException(); } + @Override + public boolean isCacheable(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + @Override public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java index e475bb6b7d9..66e0e0fe77c 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java @@ -63,11 +63,6 @@ final class ProfileScorer extends Scorer { } } - @Override - public int freq() throws IOException { - return scorer.freq(); - } - @Override public Weight getWeight() { return profileWeight; diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index bd5fd5e23a2..ae4481949ff 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -118,4 +118,9 @@ public final class ProfileWeight extends Weight { subQueryWeight.extractTerms(set); } + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return subQueryWeight.isCacheable(ctx); + } + } diff --git a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java index 796bcb2dac8..c1aaad04d1d 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java @@ -77,6 +77,11 @@ public final class DocValuesSliceQuery extends SliceQuery { return new ConstantScoreScorer(this, score(), twoPhase); } + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return DocValues.isCacheable(ctx, getField()); + } + }; } } diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java index bb6908c99fd..da1b98822cf 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java @@ -63,6 +63,11 @@ public final class TermsSliceQuery extends SliceQuery { final DocIdSetIterator leafIt = disi.iterator(); return new ConstantScoreScorer(this, score(), leafIt); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } }; } diff --git a/core/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java b/core/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java index 8e5d5b027be..4b4923ab1f8 100644 --- a/core/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java +++ b/core/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java @@ -43,7 +43,7 @@ import java.util.zip.DeflaterOutputStream; * {@link CompressibleBytesOutputStream#close()} should be called when the bytes are no longer needed and * can be safely released. */ -final class CompressibleBytesOutputStream extends StreamOutput implements Releasable { +final class CompressibleBytesOutputStream extends StreamOutput { private final StreamOutput stream; private final BytesStream bytesStreamOutput; @@ -92,18 +92,18 @@ final class CompressibleBytesOutputStream extends StreamOutput implements Releas } @Override - public void close() { + public void close() throws IOException { if (stream == bytesStreamOutput) { assert shouldCompress == false : "If the streams are the same we should not be compressing"; - IOUtils.closeWhileHandlingException(stream); + IOUtils.close(stream); } else { assert shouldCompress : "If the streams are different we should be compressing"; - IOUtils.closeWhileHandlingException(stream, bytesStreamOutput); + IOUtils.close(stream, bytesStreamOutput); } } @Override public void reset() throws IOException { - stream.reset(); + throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java index bfcd3dff5ab..6b532a600a1 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -49,7 +49,6 @@ import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.network.NetworkAddress; @@ -73,8 +72,10 @@ import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; +import java.io.Closeable; import java.io.IOException; import java.io.StreamCorruptedException; +import java.io.UncheckedIOException; import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -1704,29 +1705,36 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private final class SendListener extends SendMetricListener { private final TcpChannel channel; - private final Releasable optionalReleasable; + private final Closeable optionalCloseable; private final Runnable transportAdaptorCallback; - private SendListener(TcpChannel channel, Releasable optionalReleasable, Runnable transportAdaptorCallback, long messageLength) { + private SendListener(TcpChannel channel, Closeable optionalCloseable, Runnable transportAdaptorCallback, long messageLength) { super(messageLength); this.channel = channel; - this.optionalReleasable = optionalReleasable; + this.optionalCloseable = optionalCloseable; this.transportAdaptorCallback = transportAdaptorCallback; } @Override protected void innerInnerOnResponse(Void v) { - release(); + closeAndCallback(null); } @Override protected void innerOnFailure(Exception e) { logger.warn(() -> new ParameterizedMessage("send message failed [channel: {}]", channel), e); - release(); + closeAndCallback(e); } - private void release() { - Releasables.close(optionalReleasable, transportAdaptorCallback::run); + private void closeAndCallback(final Exception e) { + try { + IOUtils.close(optionalCloseable, transportAdaptorCallback::run); + } catch (final IOException inner) { + if (e != null) { + inner.addSuppressed(e); + } + throw new UncheckedIOException(inner); + } } } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java index a70d96a302c..8598c576c23 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java @@ -45,7 +45,6 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.hasToString; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -690,6 +689,26 @@ public class BootstrapChecksTests extends ESTestCase { BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonJava8Check), "testG1GCCheck"); } + public void testAllPermissionCheck() throws NodeValidationException { + final AtomicBoolean isAllPermissionGranted = new AtomicBoolean(true); + final BootstrapChecks.AllPermissionCheck allPermissionCheck = new BootstrapChecks.AllPermissionCheck() { + @Override + boolean isAllPermissionGranted() { + return isAllPermissionGranted.get(); + } + }; + + final List checks = Collections.singletonList(allPermissionCheck); + final NodeValidationException e = expectThrows( + NodeValidationException.class, + () -> BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck")); + assertThat(e, hasToString(containsString("granting the all permission effectively disables security"))); + + // if all permissions are not granted, nothing should happen + isAllPermissionGranted.set(false); + BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck"); + } + public void testAlwaysEnforcedChecks() { final BootstrapCheck check = new BootstrapCheck() { @Override diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java index e3c5c254d32..2b2437eea65 100644 --- a/core/src/test/java/org/elasticsearch/cli/CommandTests.java +++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java @@ -28,7 +28,7 @@ public class CommandTests extends ESTestCase { static class UserErrorCommand extends Command { UserErrorCommand() { - super("Throws a user error"); + super("Throws a user error", () -> {}); } @Override @@ -46,7 +46,7 @@ public class CommandTests extends ESTestCase { static class UsageErrorCommand extends Command { UsageErrorCommand() { - super("Throws a usage error"); + super("Throws a usage error", () -> {}); } @Override @@ -66,7 +66,7 @@ public class CommandTests extends ESTestCase { boolean executed = false; NoopCommand() { - super("Does nothing"); + super("Does nothing", () -> {}); } @Override diff --git a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java index f4680492028..f4448bbedfe 100644 --- a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java +++ b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java @@ -26,13 +26,13 @@ public class MultiCommandTests extends CommandTestCase { static class DummyMultiCommand extends MultiCommand { DummyMultiCommand() { - super("A dummy multi command"); + super("A dummy multi command", () -> {}); } } static class DummySubCommand extends Command { DummySubCommand() { - super("A dummy subcommand"); + super("A dummy subcommand", () -> {}); } @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java index de7a32b2357..6ebb604725d 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java @@ -103,11 +103,6 @@ public class MinScoreScorerTests extends LuceneTestCase { final int idx = Arrays.binarySearch(docs, docID()); return scores[idx]; } - - @Override - public int freq() throws IOException { - return 1; - } }; } diff --git a/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java b/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java index 689ecea9f4a..5d4385cbd38 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java @@ -60,23 +60,20 @@ public class CombinedDeletionPolicyTests extends ESTestCase { EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG); List commitList = new ArrayList<>(); long count = randomIntBetween(10, 20); - long minGen = Long.MAX_VALUE; + long lastGen = 0; for (int i = 0; i < count; i++) { - long lastGen = randomIntBetween(10, 20000); - minGen = Math.min(minGen, lastGen); + lastGen += randomIntBetween(10, 20000); commitList.add(mockIndexCommitWithTranslogGen(lastGen)); } combinedDeletionPolicy.onInit(commitList); - verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen); + verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen); commitList.clear(); - minGen = Long.MAX_VALUE; for (int i = 0; i < count; i++) { - long lastGen = randomIntBetween(10, 20000); - minGen = Math.min(minGen, lastGen); + lastGen += randomIntBetween(10, 20000); commitList.add(mockIndexCommitWithTranslogGen(lastGen)); } combinedDeletionPolicy.onCommit(commitList); - verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen); + verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen); } IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index eb0d4b8afa2..c40923e3c7c 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -174,7 +174,7 @@ public class InternalEngineTests extends EngineTestCase { public void testSegments() throws Exception { try (Store store = createStore(); - Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { + InternalEngine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { List segments = engine.segments(false); assertThat(segments.isEmpty(), equalTo(true)); assertThat(engine.segmentsStats(false).getCount(), equalTo(0L)); @@ -290,6 +290,69 @@ public class InternalEngineTests extends EngineTestCase { assertThat(segments.get(2).getNumDocs(), equalTo(1)); assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); assertThat(segments.get(2).isCompound(), equalTo(true)); + + // internal refresh - lets make sure we see those segments in the stats + ParsedDocument doc5 = testParsedDocument("5", null, testDocumentWithTextField(), B_3, null); + engine.index(indexForDoc(doc5)); + engine.refresh("test", Engine.SearcherScope.INTERNAL); + + segments = engine.segments(false); + assertThat(segments.size(), equalTo(4)); + assertThat(engine.segmentsStats(false).getCount(), equalTo(4L)); + assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); + assertThat(segments.get(0).isCommitted(), equalTo(true)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(1)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); + assertThat(segments.get(0).isCompound(), equalTo(true)); + + assertThat(segments.get(1).isCommitted(), equalTo(false)); + assertThat(segments.get(1).isSearch(), equalTo(true)); + assertThat(segments.get(1).getNumDocs(), equalTo(1)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(1).isCompound(), equalTo(true)); + + assertThat(segments.get(2).isCommitted(), equalTo(false)); + assertThat(segments.get(2).isSearch(), equalTo(true)); + assertThat(segments.get(2).getNumDocs(), equalTo(1)); + assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(2).isCompound(), equalTo(true)); + + assertThat(segments.get(3).isCommitted(), equalTo(false)); + assertThat(segments.get(3).isSearch(), equalTo(false)); + assertThat(segments.get(3).getNumDocs(), equalTo(1)); + assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(3).isCompound(), equalTo(true)); + + // now refresh the external searcher and make sure it has the new segment + engine.refresh("test"); + segments = engine.segments(false); + assertThat(segments.size(), equalTo(4)); + assertThat(engine.segmentsStats(false).getCount(), equalTo(4L)); + assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); + assertThat(segments.get(0).isCommitted(), equalTo(true)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(1)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); + assertThat(segments.get(0).isCompound(), equalTo(true)); + + assertThat(segments.get(1).isCommitted(), equalTo(false)); + assertThat(segments.get(1).isSearch(), equalTo(true)); + assertThat(segments.get(1).getNumDocs(), equalTo(1)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(1).isCompound(), equalTo(true)); + + assertThat(segments.get(2).isCommitted(), equalTo(false)); + assertThat(segments.get(2).isSearch(), equalTo(true)); + assertThat(segments.get(2).getNumDocs(), equalTo(1)); + assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(2).isCompound(), equalTo(true)); + + assertThat(segments.get(3).isCommitted(), equalTo(false)); + assertThat(segments.get(3).isSearch(), equalTo(true)); + assertThat(segments.get(3).getNumDocs(), equalTo(1)); + assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); + assertThat(segments.get(3).isCompound(), equalTo(true)); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java index 3e805f2b8dc..acde2e65e1f 100644 --- a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java @@ -50,11 +50,6 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase replicas = shards.getReplicas(); + IndexShard replica1 = replicas.get(0); + IndexShard replica2 = replicas.get(1); + shards.syncGlobalCheckpoint(); + + logger.info("--> Isolate replica1"); + IndexRequest indexDoc1 = new IndexRequest(index.getName(), "type", "d1").source("{}", XContentType.JSON); + BulkShardRequest replicationRequest = indexOnPrimary(indexDoc1, shards.getPrimary()); + indexOnReplica(replicationRequest, replica2); + + final Translog.Operation op1; + final List initOperations = new ArrayList<>(initDocs); + try (Translog.Snapshot snapshot = replica2.getTranslog().newSnapshot()) { + assertThat(snapshot.totalOperations(), equalTo(initDocs + 1)); + for (int i = 0; i < initDocs; i++) { + Translog.Operation op = snapshot.next(); + assertThat(op, is(notNullValue())); + initOperations.add(op); + } + op1 = snapshot.next(); + assertThat(op1, notNullValue()); + assertThat(snapshot.next(), nullValue()); + assertThat(snapshot.overriddenOperations(), equalTo(0)); + } + // Make sure that replica2 receives translog ops (eg. op2) from replica1 and overwrites its stale operation (op1). + logger.info("--> Promote replica1 as the primary"); + shards.promoteReplicaToPrimary(replica1).get(); // wait until resync completed. + shards.index(new IndexRequest(index.getName(), "type", "d2").source("{}", XContentType.JSON)); + final Translog.Operation op2; + try (Translog.Snapshot snapshot = replica2.getTranslog().newSnapshot()) { + assertThat(snapshot.totalOperations(), equalTo(initDocs + 2)); + op2 = snapshot.next(); + assertThat(op2.seqNo(), equalTo(op1.seqNo())); + assertThat(op2.primaryTerm(), greaterThan(op1.primaryTerm())); + assertThat("Remaining of snapshot should contain init operations", snapshot, containsOperationsInAnyOrder(initOperations)); + assertThat(snapshot.overriddenOperations(), equalTo(1)); + } + + // Make sure that peer-recovery transfers all but non-overridden operations. + IndexShard replica3 = shards.addReplica(); + logger.info("--> Promote replica2 as the primary"); + shards.promoteReplicaToPrimary(replica2); + logger.info("--> Recover replica3 from replica2"); + recoverReplica(replica3, replica2); + try (Translog.Snapshot snapshot = replica3.getTranslog().newSnapshot()) { + assertThat(snapshot.totalOperations(), equalTo(initDocs + 1)); + assertThat(snapshot.next(), equalTo(op2)); + assertThat("Remaining of snapshot should contain init operations", snapshot, containsOperationsInAnyOrder(initOperations)); + assertThat("Peer-recovery should not send overridden operations", snapshot.overriddenOperations(), equalTo(0)); + } + // TODO: We should assert the content of shards in the ReplicationGroup. + // Without rollback replicas(current implementation), we don't have the same content across shards: + // - replica1 has {doc1} + // - replica2 has {doc1, doc2} + // - replica3 can have either {doc2} only if operation-based recovery or {doc1, doc2} if file-based recovery + } + } + /** Throws documentFailure on every indexing operation */ static class ThrowingDocumentFailureEngineFactory implements EngineFactory { final String documentFailureMessage; diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index b29ba2d9efc..7c38b7c211f 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -18,14 +18,14 @@ */ package org.elasticsearch.index.shard; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterInfoService; @@ -41,11 +41,11 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -56,11 +56,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.flush.FlushStats; -import org.elasticsearch.index.mapper.IdFieldMapper; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesService; @@ -82,8 +77,10 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; @@ -97,6 +94,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; import static org.hamcrest.Matchers.equalTo; public class IndexShardIT extends ESSingleNodeTestCase { @@ -106,21 +104,6 @@ public class IndexShardIT extends ESSingleNodeTestCase { return pluginList(InternalSettingsPlugin.class); } - private ParsedDocument testParsedDocument(String id, String type, String routing, long seqNo, - ParseContext.Document document, BytesReference source, XContentType xContentType, - Mapping mappingUpdate) { - Field uidField = new Field("_id", id, IdFieldMapper.Defaults.FIELD_TYPE); - Field versionField = new NumericDocValuesField("_version", 0); - SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); - document.add(uidField); - document.add(versionField); - document.add(seqID.seqNo); - document.add(seqID.seqNoDocValue); - document.add(seqID.primaryTerm); - return new ParsedDocument(versionField, seqID, id, type, routing, - Collections.singletonList(document), source, xContentType, mappingUpdate); - } - public void testLockTryingToDelete() throws Exception { createIndex("test"); ensureGreen(); @@ -550,4 +533,96 @@ public class IndexShardIT extends ESSingleNodeTestCase { RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); return shardRouting; } + + public void testAutomaticRefresh() throws InterruptedException { + TimeValue randomTimeValue = randomFrom(random(), null, TimeValue.ZERO, TimeValue.timeValueMillis(randomIntBetween(0, 1000))); + Settings.Builder builder = Settings.builder(); + if (randomTimeValue != null) { + builder.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), randomTimeValue); + } + IndexService indexService = createIndex("test", builder.build()); + assertFalse(indexService.getIndexSettings().isExplicitRefresh()); + ensureGreen(); + AtomicInteger totalNumDocs = new AtomicInteger(Integer.MAX_VALUE); + CountDownLatch started = new CountDownLatch(1); + Thread t = new Thread(() -> { + SearchResponse searchResponse; + started.countDown(); + do { + searchResponse = client().prepareSearch().get(); + } while (searchResponse.getHits().totalHits != totalNumDocs.get()); + }); + t.start(); + started.await(); + assertNoSearchHits(client().prepareSearch().get()); + int numDocs = scaledRandomIntBetween(25, 100); + totalNumDocs.set(numDocs); + CountDownLatch indexingDone = new CountDownLatch(numDocs); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + indexingDone.countDown(); // one doc is indexed above blocking + IndexShard shard = indexService.getShard(0); + boolean hasRefreshed = shard.scheduledRefresh(); + if (randomTimeValue == TimeValue.ZERO) { + // with ZERO we are guaranteed to see the doc since we will wait for a refresh in the background + assertFalse(hasRefreshed); + assertTrue(shard.isSearchIdle()); + } else if (randomTimeValue == null){ + // with null we are guaranteed to see the doc since do execute the refresh. + // we can't assert on hasRefreshed since it might have been refreshed in the background on the shard concurrently + assertFalse(shard.isSearchIdle()); + } + assertHitCount(client().prepareSearch().get(), 1); + for (int i = 1; i < numDocs; i++) { + client().prepareIndex("test", "test", "" + i).setSource("{\"foo\" : \"bar\"}", XContentType.JSON) + .execute(new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + indexingDone.countDown(); + } + + @Override + public void onFailure(Exception e) { + indexingDone.countDown(); + throw new AssertionError(e); + } + }); + } + indexingDone.await(); + t.join(); + } + + public void testPendingRefreshWithIntervalChange() throws InterruptedException { + Settings.Builder builder = Settings.builder(); + builder.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO); + IndexService indexService = createIndex("test", builder.build()); + assertFalse(indexService.getIndexSettings().isExplicitRefresh()); + ensureGreen(); + assertNoSearchHits(client().prepareSearch().get()); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + IndexShard shard = indexService.getShard(0); + assertFalse(shard.scheduledRefresh()); + assertTrue(shard.isSearchIdle()); + CountDownLatch refreshLatch = new CountDownLatch(1); + client().admin().indices().prepareRefresh() + .execute(ActionListener.wrap(refreshLatch::countDown));// async on purpose to make sure it happens concurrently + assertHitCount(client().prepareSearch().get(), 1); + client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + assertFalse(shard.scheduledRefresh()); + + // now disable background refresh and make sure the refresh happens + CountDownLatch updateSettingsLatch = new CountDownLatch(1); + client().admin().indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1).build()) + .execute(ActionListener.wrap(updateSettingsLatch::countDown)); + assertHitCount(client().prepareSearch().get(), 2); + // wait for both to ensure we don't have in-flight operations + updateSettingsLatch.await(); + refreshLatch.await(); + + client().prepareIndex("test", "test", "2").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get(); + assertTrue(shard.scheduledRefresh()); + assertTrue(shard.isSearchIdle()); + assertHitCount(client().prepareSearch().get(), 3); + } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 89e2f844174..fe6d4ccdea0 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -62,7 +62,9 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -70,6 +72,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineException; @@ -2269,11 +2272,17 @@ public class IndexShardTests extends IndexShardTestCase { final String id = Integer.toString(i); indexDoc(indexShard, "test", id); } - - indexShard.refresh("test"); + if (randomBoolean()) { + indexShard.refresh("test"); + } else { + indexShard.flush(new FlushRequest()); + } { final DocsStats docsStats = indexShard.docStats(); assertThat(docsStats.getCount(), equalTo(numDocs)); + try (Engine.Searcher searcher = indexShard.acquireSearcher("test")) { + assertTrue(searcher.reader().numDocs() <= docsStats.getCount()); + } assertThat(docsStats.getDeleted(), equalTo(0L)); assertThat(docsStats.getAverageSizeInBytes(), greaterThan(0L)); } @@ -2293,9 +2302,14 @@ public class IndexShardTests extends IndexShardTestCase { flushRequest.waitIfOngoing(false); indexShard.flush(flushRequest); - indexShard.refresh("test"); + if (randomBoolean()) { + indexShard.refresh("test"); + } { final DocsStats docStats = indexShard.docStats(); + try (Engine.Searcher searcher = indexShard.acquireSearcher("test")) { + assertTrue(searcher.reader().numDocs() <= docStats.getCount()); + } assertThat(docStats.getCount(), equalTo(numDocs)); // Lucene will delete a segment if all docs are deleted from it; this means that we lose the deletes when deleting all docs assertThat(docStats.getDeleted(), equalTo(numDocsToDelete == numDocs ? 0 : numDocsToDelete)); @@ -2307,7 +2321,11 @@ public class IndexShardTests extends IndexShardTestCase { forceMergeRequest.maxNumSegments(1); indexShard.forceMerge(forceMergeRequest); - indexShard.refresh("test"); + if (randomBoolean()) { + indexShard.refresh("test"); + } else { + indexShard.flush(new FlushRequest()); + } { final DocsStats docStats = indexShard.docStats(); assertThat(docStats.getCount(), equalTo(numDocs)); @@ -2338,8 +2356,11 @@ public class IndexShardTests extends IndexShardTestCase { assertThat("Without flushing, segment sizes should be zero", indexShard.docStats().getTotalSizeInBytes(), equalTo(0L)); - indexShard.flush(new FlushRequest()); - indexShard.refresh("test"); + if (randomBoolean()) { + indexShard.flush(new FlushRequest()); + } else { + indexShard.refresh("test"); + } { final DocsStats docsStats = indexShard.docStats(); final StoreStats storeStats = indexShard.storeStats(); @@ -2359,9 +2380,11 @@ public class IndexShardTests extends IndexShardTestCase { indexDoc(indexShard, "doc", Integer.toString(i), "{\"foo\": \"bar\"}"); } } - - indexShard.flush(new FlushRequest()); - indexShard.refresh("test"); + if (randomBoolean()) { + indexShard.flush(new FlushRequest()); + } else { + indexShard.refresh("test"); + } { final DocsStats docsStats = indexShard.docStats(); final StoreStats storeStats = indexShard.storeStats(); @@ -2565,4 +2588,138 @@ public class IndexShardTests extends IndexShardTestCase { public void verify(String verificationToken, DiscoveryNode localNode) { } } + + public void testIsSearchIdle() throws Exception { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("test") + .putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") + .settings(settings) + .primaryTerm(0, 1).build(); + IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(primary); + indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}"); + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + assertFalse(primary.isSearchIdle()); + + IndexScopedSettings scopedSettings = primary.indexSettings().getScopedSettings(); + settings = Settings.builder().put(settings).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO).build(); + scopedSettings.applySettings(settings); + assertTrue(primary.isSearchIdle()); + + settings = Settings.builder().put(settings).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMinutes(1)) + .build(); + scopedSettings.applySettings(settings); + assertFalse(primary.isSearchIdle()); + + settings = Settings.builder().put(settings).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMillis(10)) + .build(); + scopedSettings.applySettings(settings); + + assertBusy(() -> assertTrue(primary.isSearchIdle())); + do { + // now loop until we are fast enough... shouldn't take long + primary.awaitShardSearchActive(aBoolean -> {}); + } while (primary.isSearchIdle()); + closeShards(primary); + } + + public void testScheduledRefresh() throws IOException, InterruptedException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("test") + .putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") + .settings(settings) + .primaryTerm(0, 1).build(); + IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(primary); + indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}"); + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + IndexScopedSettings scopedSettings = primary.indexSettings().getScopedSettings(); + settings = Settings.builder().put(settings).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO).build(); + scopedSettings.applySettings(settings); + + assertFalse(primary.getEngine().refreshNeeded()); + indexDoc(primary, "test", "1", "{\"foo\" : \"bar\"}"); + assertTrue(primary.getEngine().refreshNeeded()); + long lastSearchAccess = primary.getLastSearcherAccess(); + assertFalse(primary.scheduledRefresh()); + assertEquals(lastSearchAccess, primary.getLastSearcherAccess()); + // wait until the thread-pool has moved the timestamp otherwise we can't assert on this below + awaitBusy(() -> primary.getThreadPool().relativeTimeInMillis() > lastSearchAccess); + CountDownLatch latch = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + primary.awaitShardSearchActive(refreshed -> { + assertTrue(refreshed); + try (Engine.Searcher searcher = primary.acquireSearcher("test")) { + assertEquals(2, searcher.reader().numDocs()); + } finally { + latch.countDown(); + } + }); + } + assertNotEquals("awaitShardSearchActive must access a searcher to remove search idle state", lastSearchAccess, + primary.getLastSearcherAccess()); + assertTrue(lastSearchAccess < primary.getLastSearcherAccess()); + try (Engine.Searcher searcher = primary.acquireSearcher("test")) { + assertEquals(1, searcher.reader().numDocs()); + } + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + latch.await(); + CountDownLatch latch1 = new CountDownLatch(1); + primary.awaitShardSearchActive(refreshed -> { + assertFalse(refreshed); + try (Engine.Searcher searcher = primary.acquireSearcher("test")) { + assertEquals(2, searcher.reader().numDocs()); + } finally { + latch1.countDown(); + } + + }); + latch1.await(); + closeShards(primary); + } + + public void testRefreshIsNeededWithRefreshListeners() throws IOException, InterruptedException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("test") + .putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") + .settings(settings) + .primaryTerm(0, 1).build(); + IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(primary); + indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}"); + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + Engine.IndexResult doc = indexDoc(primary, "test", "1", "{\"foo\" : \"bar\"}"); + CountDownLatch latch = new CountDownLatch(1); + primary.addRefreshListener(doc.getTranslogLocation(), r -> latch.countDown()); + assertEquals(1, latch.getCount()); + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + latch.await(); + + IndexScopedSettings scopedSettings = primary.indexSettings().getScopedSettings(); + settings = Settings.builder().put(settings).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.ZERO).build(); + scopedSettings.applySettings(settings); + + doc = indexDoc(primary, "test", "2", "{\"foo\" : \"bar\"}"); + CountDownLatch latch1 = new CountDownLatch(1); + primary.addRefreshListener(doc.getTranslogLocation(), r -> latch1.countDown()); + assertEquals(1, latch1.getCount()); + assertTrue(primary.getEngine().refreshNeeded()); + assertTrue(primary.scheduledRefresh()); + latch1.await(); + closeShards(primary); + } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index fcc3c93fc37..53ced098c04 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -67,6 +68,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -87,16 +89,16 @@ public class RefreshListenersTests extends ESTestCase { public void setupListeners() throws Exception { // Setup dependencies of the listeners maxListeners = randomIntBetween(1, 1000); + // Now setup the InternalEngine which is much more complicated because we aren't mocking anything + threadPool = new TestThreadPool(getTestName()); listeners = new RefreshListeners( () -> maxListeners, () -> engine.refresh("too-many-listeners"), // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test. Runnable::run, - logger - ); + logger, + threadPool.getThreadContext()); - // Now setup the InternalEngine which is much more complicated because we aren't mocking anything - threadPool = new TestThreadPool(getTestName()); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY); ShardId shardId = new ShardId(new Index("index", "_na_"), 1); String allocationId = UUIDs.randomBase64UUID(random()); @@ -161,6 +163,23 @@ public class RefreshListenersTests extends ESTestCase { assertEquals(0, listeners.pendingCount()); } + public void testContextIsPreserved() throws IOException, InterruptedException { + assertEquals(0, listeners.pendingCount()); + Engine.IndexResult index = index("1"); + CountDownLatch latch = new CountDownLatch(1); + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().putHeader("test", "foobar"); + assertFalse(listeners.addOrNotify(index.getTranslogLocation(), forced -> { + assertEquals("foobar", threadPool.getThreadContext().getHeader("test")); + latch.countDown(); + })); + } + assertNull(threadPool.getThreadContext().getHeader("test")); + assertEquals(1, latch.getCount()); + engine.refresh("I said so"); + latch.await(); + } + public void testTooMany() throws Exception { assertEquals(0, listeners.pendingCount()); assertFalse(listeners.refreshNeeded()); diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java index 1fe6be53466..9cc6d86bc2f 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MockEngineFactoryPlugin; +import org.elasticsearch.index.translog.TestTranslog; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -43,23 +44,17 @@ import org.elasticsearch.test.engine.MockEngineSupport; import org.elasticsearch.test.transport.MockTransportService; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.notNullValue; /** * Integration test for corrupted translog files @@ -119,51 +114,14 @@ public class CorruptedTranslogIT extends ESIntegTestCase { assertTrue(shardRouting.assignedToNode()); String nodeId = shardRouting.currentNodeId(); NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get(); - Set files = new TreeSet<>(); // treeset makes sure iteration order is deterministic + Set translogDirs = new HashSet<>(); for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) { String path = fsPath.getPath(); - final String relativeDataLocationPath = "indices/"+ test.getUUID() +"/" + Integer.toString(shardRouting.getId()) + "/translog"; - Path file = PathUtils.get(path).resolve(relativeDataLocationPath); - if (Files.exists(file)) { - logger.info("--> path: {}", file); - try (DirectoryStream stream = Files.newDirectoryStream(file)) { - for (Path item : stream) { - logger.info("--> File: {}", item); - if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) { - files.add(item); - } - } - } - } + String relativeDataLocationPath = "indices/" + test.getUUID() + "/" + Integer.toString(shardRouting.getId()) + "/translog"; + Path translogDir = PathUtils.get(path).resolve(relativeDataLocationPath); + translogDirs.add(translogDir); } - Path fileToCorrupt = null; - if (!files.isEmpty()) { - int corruptions = randomIntBetween(5, 20); - for (int i = 0; i < corruptions; i++) { - fileToCorrupt = RandomPicks.randomFrom(random(), files); - try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { - // read - raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); - long filePointer = raf.position(); - ByteBuffer bb = ByteBuffer.wrap(new byte[1]); - raf.read(bb); - bb.flip(); - - // corrupt - byte oldValue = bb.get(0); - byte newValue = (byte) (oldValue + 1); - bb.put(0, newValue); - - // rewrite - raf.position(filePointer); - raf.write(bb); - logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}", - fileToCorrupt, filePointer, Integer.toHexString(oldValue), - Integer.toHexString(newValue), fileToCorrupt); - } - } - } - assertThat("no file corrupted", fileToCorrupt, notNullValue()); + TestTranslog.corruptTranslogFiles(logger, random(), translogDirs); } /** Disables translog flushing for the specified index */ diff --git a/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java b/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java new file mode 100644 index 00000000000..7ee2a6c3366 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.translog; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongSet; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class MultiSnapshotTests extends ESTestCase { + + public void testTrackSeqNoSimpleRange() throws Exception { + final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); + final List values = LongStream.range(0, 1024).boxed().collect(Collectors.toList()); + Randomness.shuffle(values); + for (int i = 0; i < 1023; i++) { + assertThat(bitSet.getAndSet(values.get(i)), equalTo(false)); + assertThat(bitSet.ongoingSetsSize(), equalTo(1L)); + assertThat(bitSet.completeSetsSize(), equalTo(0L)); + } + + assertThat(bitSet.getAndSet(values.get(1023)), equalTo(false)); + assertThat(bitSet.ongoingSetsSize(), equalTo(0L)); + assertThat(bitSet.completeSetsSize(), equalTo(1L)); + + assertThat(bitSet.getAndSet(between(0, 1023)), equalTo(true)); + assertThat(bitSet.getAndSet(between(1024, Integer.MAX_VALUE)), equalTo(false)); + } + + public void testTrackSeqNoDenseRanges() throws Exception { + final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); + final LongSet normalSet = new LongHashSet(); + IntStream.range(0, scaledRandomIntBetween(5_000, 10_000)).forEach(i -> { + long seq = between(0, 5000); + boolean existed = normalSet.add(seq) == false; + assertThat("SeqNoSet != Set" + seq, bitSet.getAndSet(seq), equalTo(existed)); + assertThat(bitSet.ongoingSetsSize() + bitSet.completeSetsSize(), lessThanOrEqualTo(5L)); + }); + } + + public void testTrackSeqNoSparseRanges() throws Exception { + final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); + final LongSet normalSet = new LongHashSet(); + IntStream.range(0, scaledRandomIntBetween(5_000, 10_000)).forEach(i -> { + long seq = between(i * 10_000, i * 30_000); + boolean existed = normalSet.add(seq) == false; + assertThat("SeqNoSet != Set", bitSet.getAndSet(seq), equalTo(existed)); + }); + } + + public void testTrackSeqNoMimicTranslogRanges() throws Exception { + final MultiSnapshot.SeqNoSet bitSet = new MultiSnapshot.SeqNoSet(); + final LongSet normalSet = new LongHashSet(); + long currentSeq = between(10_000_000, 1_000_000_000); + final int iterations = scaledRandomIntBetween(100, 2000); + assertThat(bitSet.completeSetsSize(), equalTo(0L)); + assertThat(bitSet.ongoingSetsSize(), equalTo(0L)); + long totalDocs = 0; + for (long i = 0; i < iterations; i++) { + int batchSize = between(1, 1500); + totalDocs += batchSize; + currentSeq -= batchSize; + List batch = LongStream.range(currentSeq, currentSeq + batchSize) + .boxed() + .collect(Collectors.toList()); + Randomness.shuffle(batch); + batch.forEach(seq -> { + boolean existed = normalSet.add(seq) == false; + assertThat("SeqNoSet != Set", bitSet.getAndSet(seq), equalTo(existed)); + assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(4L)); + }); + assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(2L)); + } + assertThat(bitSet.completeSetsSize(), lessThanOrEqualTo(totalDocs / 1024)); + assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(2L)); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java b/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java index c45da660b00..4ca6057bd6b 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java +++ b/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java @@ -26,6 +26,9 @@ import org.hamcrest.TypeSafeMatcher; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; public final class SnapshotMatchers { @@ -50,10 +53,14 @@ public final class SnapshotMatchers { /** * Consumes a snapshot and make sure it's content is as expected */ - public static Matcher equalsTo(ArrayList ops) { + public static Matcher equalsTo(List ops) { return new EqualMatcher(ops.toArray(new Translog.Operation[ops.size()])); } + public static Matcher containsOperationsInAnyOrder(Collection expectedOperations) { + return new ContainingInAnyOrderMatcher(expectedOperations); + } + public static class SizeMatcher extends TypeSafeMatcher { private final int size; @@ -127,5 +134,60 @@ public final class SnapshotMatchers { } } + public static class ContainingInAnyOrderMatcher extends TypeSafeMatcher { + private final Collection expectedOps; + private List notFoundOps; + private List notExpectedOps; + static List drainAll(Translog.Snapshot snapshot) throws IOException { + final List actualOps = new ArrayList<>(); + Translog.Operation op; + while ((op = snapshot.next()) != null) { + actualOps.add(op); + } + return actualOps; + } + + public ContainingInAnyOrderMatcher(Collection expectedOps) { + this.expectedOps = expectedOps; + } + + @Override + protected boolean matchesSafely(Translog.Snapshot snapshot) { + try { + List actualOps = drainAll(snapshot); + notFoundOps = expectedOps.stream() + .filter(o -> actualOps.contains(o) == false) + .collect(Collectors.toList()); + notExpectedOps = actualOps.stream() + .filter(o -> expectedOps.contains(o) == false) + .collect(Collectors.toList()); + return notFoundOps.isEmpty() && notExpectedOps.isEmpty(); + } catch (IOException ex) { + throw new ElasticsearchException("failed to read snapshot content", ex); + } + } + + @Override + protected void describeMismatchSafely(Translog.Snapshot snapshot, Description mismatchDescription) { + if (notFoundOps.isEmpty() == false) { + mismatchDescription + .appendText("not found ").appendValueList("[", ", ", "]", notFoundOps); + } + if (notExpectedOps.isEmpty() == false) { + if (notFoundOps.isEmpty() == false) { + mismatchDescription.appendText("; "); + } + mismatchDescription + .appendText("not expected ").appendValueList("[", ", ", "]", notExpectedOps); + } + } + + @Override + public void describeTo(Description description) { + description.appendText("snapshot contains ") + .appendValueList("[", ", ", "]", expectedOps) + .appendText(" in any order."); + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TestTranslog.java b/core/src/test/java/org/elasticsearch/index/translog/TestTranslog.java new file mode 100644 index 00000000000..751f989a0a6 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/translog/TestTranslog.java @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.translog; + +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collection; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.elasticsearch.index.translog.Translog.CHECKPOINT_FILE_NAME; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.not; + +/** + * Helpers for testing translog. + */ +public class TestTranslog { + static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog"); + + /** + * Corrupts some translog files (translog-N.tlog) from the given translog directories. + * + * @return a collection of tlog files that have been corrupted. + */ + public static Set corruptTranslogFiles(Logger logger, Random random, Collection translogDirs) throws IOException { + Set candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic + + for (Path translogDir : translogDirs) { + logger.info("--> Translog dir: {}", translogDir); + if (Files.isDirectory(translogDir)) { + final Checkpoint checkpoint = Checkpoint.read(translogDir.resolve(CHECKPOINT_FILE_NAME)); + final long minTranslogGeneration = checkpoint.minTranslogGeneration; + try (DirectoryStream stream = Files.newDirectoryStream(translogDir)) { + for (Path item : stream) { + if (Files.isRegularFile(item)) { + // Makes sure that we will corrupt tlog files that are referenced by the Checkpoint. + final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString()); + if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minTranslogGeneration) { + candidates.add(item); + } + } + } + } + } + } + + Set corruptedFiles = new HashSet<>(); + if (!candidates.isEmpty()) { + int corruptions = RandomNumbers.randomIntBetween(random, 5, 20); + for (int i = 0; i < corruptions; i++) { + Path fileToCorrupt = RandomPicks.randomFrom(random, candidates); + try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + // read + raf.position(RandomNumbers.randomIntBetween(random, 0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); + long filePointer = raf.position(); + ByteBuffer bb = ByteBuffer.wrap(new byte[1]); + raf.read(bb); + bb.flip(); + + // corrupt + byte oldValue = bb.get(0); + byte newValue = (byte) (oldValue + 1); + bb.put(0, newValue); + + // rewrite + raf.position(filePointer); + raf.write(bb); + logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}", + fileToCorrupt, filePointer, Integer.toHexString(oldValue), + Integer.toHexString(newValue), fileToCorrupt); + } + corruptedFiles.add(fileToCorrupt); + } + } + assertThat("no translog file corrupted", corruptedFiles, not(empty())); + return corruptedFiles; + } +} diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 1a17e0dc6a0..593e1059215 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -83,10 +83,13 @@ import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -107,6 +110,7 @@ import java.util.stream.LongStream; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomLongBetween; import static org.elasticsearch.common.util.BigArrays.NON_RECYCLING_INSTANCE; +import static org.elasticsearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder; import static org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -217,7 +221,7 @@ public class TranslogTests extends ESTestCase { return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize); } - private void addToTranslogAndList(Translog translog, ArrayList list, Translog.Operation op) throws IOException { + private void addToTranslogAndList(Translog translog, List list, Translog.Operation op) throws IOException { list.add(op); translog.add(op); } @@ -520,7 +524,7 @@ public class TranslogTests extends ESTestCase { Translog.Snapshot snapshot2 = translog.newSnapshot(); toClose.add(snapshot2); markCurrentGenAsCommitted(translog); - assertThat(snapshot2, SnapshotMatchers.equalsTo(ops)); + assertThat(snapshot2, containsOperationsInAnyOrder(ops)); assertThat(snapshot2.totalOperations(), equalTo(ops.size())); } finally { IOUtils.closeWhileHandlingException(toClose); @@ -1028,7 +1032,7 @@ public class TranslogTests extends ESTestCase { } assertEquals(max.generation, translog.currentFileGeneration()); - try (Translog.Snapshot snap = translog.newSnapshot()) { + try (Translog.Snapshot snap = new SortedSnapshot(translog.newSnapshot())) { Translog.Operation next; Translog.Operation maxOp = null; while ((next = snap.next()) != null) { @@ -1252,7 +1256,7 @@ public class TranslogTests extends ESTestCase { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); - try (Translog.Snapshot snapshot = translog.newSnapshot()) { + try (Translog.Snapshot snapshot = new SortedSnapshot(translog.newSnapshot())) { int upTo = sync ? translogOperations : prepareOp; for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); @@ -1266,7 +1270,7 @@ public class TranslogTests extends ESTestCase { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice", translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); - try (Translog.Snapshot snapshot = translog.newSnapshot()) { + try (Translog.Snapshot snapshot = new SortedSnapshot(translog.newSnapshot())) { int upTo = sync ? translogOperations : prepareOp; for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); @@ -1310,7 +1314,7 @@ public class TranslogTests extends ESTestCase { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); - try (Translog.Snapshot snapshot = translog.newSnapshot()) { + try (Translog.Snapshot snapshot = new SortedSnapshot(translog.newSnapshot())) { int upTo = sync ? translogOperations : prepareOp; for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); @@ -1325,7 +1329,7 @@ public class TranslogTests extends ESTestCase { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice", translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); - try (Translog.Snapshot snapshot = translog.newSnapshot()) { + try (Translog.Snapshot snapshot = new SortedSnapshot(translog.newSnapshot())) { int upTo = sync ? translogOperations : prepareOp; for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); @@ -1374,7 +1378,7 @@ public class TranslogTests extends ESTestCase { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); - try (Translog.Snapshot snapshot = translog.newSnapshot()) { + try (Translog.Snapshot snapshot = new SortedSnapshot(translog.newSnapshot())) { int upTo = sync ? translogOperations : prepareOp; for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); @@ -2061,7 +2065,7 @@ public class TranslogTests extends ESTestCase { } public void testRecoverWithUnbackedNextGen() throws IOException { - translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); translog.close(); TranslogConfig config = translog.getConfig(); @@ -2072,21 +2076,25 @@ public class TranslogTests extends ESTestCase { try (Translog tlog = createTranslog(config, translog.getTranslogUUID()); Translog.Snapshot snapshot = tlog.newSnapshot()) { assertFalse(tlog.syncNeeded()); - for (int i = 0; i < 1; i++) { - Translog.Operation next = snapshot.next(); - assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); - } - tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + + Translog.Operation op = snapshot.next(); + assertNotNull("operation 1 must be non-null", op); + assertEquals("payload mismatch for operation 1", 1, Integer.parseInt(op.getSource().source.utf8ToString())); + + tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(2).getBytes(Charset.forName("UTF-8")))); } + try (Translog tlog = createTranslog(config, translog.getTranslogUUID()); Translog.Snapshot snapshot = tlog.newSnapshot()) { assertFalse(tlog.syncNeeded()); - for (int i = 0; i < 2; i++) { - Translog.Operation next = snapshot.next(); - assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); - } + + Translog.Operation secondOp = snapshot.next(); + assertNotNull("operation 2 must be non-null", secondOp); + assertEquals("payload mismatch for operation 2", Integer.parseInt(secondOp.getSource().source.utf8ToString()), 2); + + Translog.Operation firstOp = snapshot.next(); + assertNotNull("operation 1 must be non-null", firstOp); + assertEquals("payload mismatch for operation 1", Integer.parseInt(firstOp.getSource().source.utf8ToString()), 1); } } @@ -2489,6 +2497,7 @@ public class TranslogTests extends ESTestCase { assertThat(Tuple.tuple(op.seqNo(), op.primaryTerm()), isIn(seenSeqNos)); readFromSnapshot++; } + readFromSnapshot += snapshot.overriddenOperations(); } assertThat(readFromSnapshot, equalTo(expectedSnapshotOps)); final long seqNoLowerBound = seqNo; @@ -2534,4 +2543,84 @@ public class TranslogTests extends ESTestCase { } } } + + public void testSnapshotReadOperationInReverse() throws Exception { + final Deque> views = new ArrayDeque<>(); + views.push(new ArrayList<>()); + final AtomicLong seqNo = new AtomicLong(); + + final int generations = randomIntBetween(2, 20); + for (int gen = 0; gen < generations; gen++) { + final int operations = randomIntBetween(1, 100); + for (int i = 0; i < operations; i++) { + Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo.getAndIncrement(), new byte[]{1}); + translog.add(op); + views.peek().add(op); + } + if (frequently()) { + translog.rollGeneration(); + views.push(new ArrayList<>()); + } + } + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + final List expectedSeqNo = new ArrayList<>(); + while (views.isEmpty() == false) { + expectedSeqNo.addAll(views.pop()); + } + assertThat(snapshot, SnapshotMatchers.equalsTo(expectedSeqNo)); + } + } + + public void testSnapshotDedupOperations() throws Exception { + final Map latestOperations = new HashMap<>(); + final int generations = between(2, 20); + for (int gen = 0; gen < generations; gen++) { + List batch = LongStream.rangeClosed(0, between(0, 500)).boxed().collect(Collectors.toList()); + Randomness.shuffle(batch); + for (Long seqNo : batch) { + Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo, new byte[]{1}); + translog.add(op); + latestOperations.put(op.seqNo(), op); + } + translog.rollGeneration(); + } + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + assertThat(snapshot, containsOperationsInAnyOrder(latestOperations.values())); + } + } + + static class SortedSnapshot implements Translog.Snapshot { + private final Translog.Snapshot snapshot; + private List operations = null; + + SortedSnapshot(Translog.Snapshot snapshot) { + this.snapshot = snapshot; + } + + @Override + public int totalOperations() { + return snapshot.totalOperations(); + } + + @Override + public Translog.Operation next() throws IOException { + if (operations == null) { + operations = new ArrayList<>(); + Translog.Operation op; + while ((op = snapshot.next()) != null) { + operations.add(op); + } + operations.sort(Comparator.comparing(Translog.Operation::seqNo)); + } + if (operations.isEmpty()) { + return null; + } + return operations.remove(0); + } + + @Override + public void close() throws IOException { + snapshot.close(); + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java index b0d4c238679..d98359cdd06 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java @@ -60,12 +60,8 @@ import org.elasticsearch.test.engine.MockEngineSupport; import org.elasticsearch.test.transport.MockTransportService; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -79,7 +75,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0) public class TruncateTranslogIT extends ESIntegTestCase { @@ -273,7 +268,7 @@ public class TruncateTranslogIT extends ESIntegTestCase { // Corrupt the translog file(s) logger.info("--> corrupting translog"); - corruptTranslogFiles(translogDirs); + TestTranslog.corruptTranslogFiles(logger, random(), translogDirs); // Restart the single node logger.info("--> starting node"); @@ -354,52 +349,7 @@ public class TruncateTranslogIT extends ESIntegTestCase { private void corruptRandomTranslogFiles(String indexName) throws IOException { Set translogDirs = getTranslogDirs(indexName); - corruptTranslogFiles(translogDirs); - } - - private void corruptTranslogFiles(Set translogDirs) throws IOException { - Set files = new TreeSet<>(); // treeset makes sure iteration order is deterministic - for (Path translogDir : translogDirs) { - if (Files.isDirectory(translogDir)) { - logger.info("--> path: {}", translogDir); - try (DirectoryStream stream = Files.newDirectoryStream(translogDir)) { - for (Path item : stream) { - logger.info("--> File: {}", item); - if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) { - files.add(item); - } - } - } - } - } - Path fileToCorrupt = null; - if (!files.isEmpty()) { - int corruptions = randomIntBetween(5, 20); - for (int i = 0; i < corruptions; i++) { - fileToCorrupt = RandomPicks.randomFrom(random(), files); - try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { - // read - raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); - long filePointer = raf.position(); - ByteBuffer bb = ByteBuffer.wrap(new byte[1]); - raf.read(bb); - bb.flip(); - - // corrupt - byte oldValue = bb.get(0); - byte newValue = (byte) (oldValue + 1); - bb.put(0, newValue); - - // rewrite - raf.position(filePointer); - raf.write(bb); - logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}", - fileToCorrupt, filePointer, Integer.toHexString(oldValue), - Integer.toHexString(newValue), fileToCorrupt); - } - } - } - assertThat("no file corrupted", fileToCorrupt, notNullValue()); + TestTranslog.corruptTranslogFiles(logger, random(), translogDirs); } /** Disables translog flushing for the specified index */ diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 24cbed2d4bc..d8af4ad00e0 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -79,6 +79,11 @@ public class IndicesQueryCacheTests extends ESTestCase { public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } }; } @@ -364,6 +369,11 @@ public class IndicesQueryCacheTests extends ESTestCase { return weight.scorerSupplier(context); } + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + } public void testDelegatesScorerSupplier() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java index b16aacd9a16..bd6298f5d03 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java @@ -30,6 +30,7 @@ import java.util.List; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; @@ -52,7 +53,7 @@ public class AggregationsIntegrationIT extends ESIntegTestCase { public void testScroll() { final int size = randomIntBetween(1, 4); SearchResponse response = client().prepareSearch("index") - .setSize(size).setScroll(new TimeValue(500)) + .setSize(size).setScroll(TimeValue.timeValueMinutes(1)) .addAggregation(terms("f").field("f")).get(); assertSearchResponse(response); Aggregations aggregations = response.getAggregations(); @@ -63,8 +64,9 @@ public class AggregationsIntegrationIT extends ESIntegTestCase { int total = response.getHits().getHits().length; while (response.getHits().getHits().length > 0) { response = client().prepareSearchScroll(response.getScrollId()) - .setScroll(new TimeValue(500)) + .setScroll(TimeValue.timeValueMinutes(1)) .execute().actionGet(); + assertSearchResponse(response); assertNull(response.getAggregations()); total += response.getHits().getHits().length; } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java similarity index 98% rename from core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java index b27a509b8ec..3b7e686ef4d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.aggregations.bucket; +package org.elasticsearch.search.aggregations.bucket.terms; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -34,9 +34,11 @@ import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.search.aggregations.metrics.stats.Stats; @@ -44,6 +46,8 @@ import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStat import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; @@ -84,6 +88,18 @@ public class StringTermsIT extends AbstractTermsTestCase { return Collections.singleton(CustomScriptPlugin.class); } + @Before + public void randomizeOptimizations() { + TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = false;randomBoolean(); + TermsAggregatorFactory.REMAP_GLOBAL_ORDS = randomBoolean(); + } + + @After + public void resetOptimizations() { + TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = null; + TermsAggregatorFactory.REMAP_GLOBAL_ORDS = null; + } + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { @Override diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 266b1a6e50f..47fccbc83c4 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -38,6 +38,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -68,7 +69,27 @@ import java.util.function.Function; import static org.hamcrest.Matchers.instanceOf; public class TermsAggregatorTests extends AggregatorTestCase { + + private boolean randomizeAggregatorImpl = true; + + @Override + protected A createAggregator(AggregationBuilder aggregationBuilder, + IndexSearcher indexSearcher, IndexSettings indexSettings, MappedFieldType... fieldTypes) throws IOException { + try { + if (randomizeAggregatorImpl) { + TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = randomBoolean(); + TermsAggregatorFactory.REMAP_GLOBAL_ORDS = randomBoolean(); + } + return super.createAggregator(aggregationBuilder, indexSearcher, indexSettings, fieldTypes); + } finally { + TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = null; + TermsAggregatorFactory.REMAP_GLOBAL_ORDS = null; + } + } + public void testGlobalOrdinalsExecutionHint() throws Exception { + randomizeAggregatorImpl = false; + Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); indexWriter.close(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 568b8e7996f..fb18cd99032 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.function.LongUnaryOperator; public class MissingValuesTests extends ESTestCase { @@ -111,7 +112,7 @@ public class MissingValuesTests extends ESTestCase { ords[i][j] = TestUtil.nextInt(random(), ords[i][j], maxOrd - 1); } } - SortedSetDocValues asRandomAccessOrds = new AbstractSortedSetDocValues() { + SortedSetDocValues asSortedSet = new AbstractSortedSetDocValues() { int doc = -1; int i; @@ -147,7 +148,7 @@ public class MissingValuesTests extends ESTestCase { final BytesRef missingMissing = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 5)); for (BytesRef missing : Arrays.asList(existingMissing, missingMissing)) { - SortedSetDocValues withMissingReplaced = MissingValues.replaceMissing(asRandomAccessOrds, missing); + SortedSetDocValues withMissingReplaced = MissingValues.replaceMissing(asSortedSet, missing); if (valueSet.contains(missing)) { assertEquals(values.length, withMissingReplaced.getValueCount()); } else { @@ -169,6 +170,84 @@ public class MissingValuesTests extends ESTestCase { } } + public void testGlobalMapping() throws IOException { + final int numOrds = TestUtil.nextInt(random(), 1, 10); + final int numGlobalOrds = TestUtil.nextInt(random(), numOrds, numOrds + 3); + + final Set valueSet = new HashSet<>(); + while (valueSet.size() < numOrds) { + valueSet.add(new BytesRef(RandomStrings.randomAsciiLettersOfLength(random(), 5))); + } + final BytesRef[] values = valueSet.toArray(new BytesRef[0]); + Arrays.sort(values); + + final Set globalValueSet = new HashSet<>(valueSet); + while (globalValueSet.size() < numGlobalOrds) { + globalValueSet.add(new BytesRef(RandomStrings.randomAsciiLettersOfLength(random(), 5))); + } + final BytesRef[] globalValues = globalValueSet.toArray(new BytesRef[0]); + Arrays.sort(globalValues); + + // exists in the current segment + BytesRef missing = RandomPicks.randomFrom(random(), values); + doTestGlobalMapping(values, globalValues, missing); + + // missing in all segments + do { + missing = new BytesRef(RandomStrings.randomAsciiLettersOfLength(random(), 5)); + } while (globalValueSet.contains(missing)); + doTestGlobalMapping(values, globalValues, missing); + + if (globalValueSet.size() > valueSet.size()) { + // exists in other segments only + Set other = new HashSet<>(globalValueSet); + other.removeAll(valueSet); + missing = RandomPicks.randomFrom(random(), other.toArray(new BytesRef[0])); + doTestGlobalMapping(values, globalValues, missing); + } + } + + private void doTestGlobalMapping(BytesRef[] values, BytesRef[] globalValues, BytesRef missing) throws IOException { + LongUnaryOperator segmentToGlobalOrd = segmentOrd -> Arrays.binarySearch(globalValues, values[Math.toIntExact(segmentOrd)]); + SortedSetDocValues sortedValues = asOrds(values); + SortedSetDocValues sortedGlobalValues = asOrds(globalValues); + + LongUnaryOperator withMissingSegmentToGlobalOrd = MissingValues.getGlobalMapping( + sortedValues, sortedGlobalValues, segmentToGlobalOrd, missing); + SortedSetDocValues withMissingValues = MissingValues.replaceMissing(sortedValues, missing); + SortedSetDocValues withMissingGlobalValues = MissingValues.replaceMissing(sortedGlobalValues, missing); + + for (long segmentOrd = 0; segmentOrd < withMissingValues.getValueCount(); ++segmentOrd) { + long expectedGlobalOrd = withMissingSegmentToGlobalOrd.applyAsLong(segmentOrd); + assertEquals(withMissingValues.lookupOrd(segmentOrd), withMissingGlobalValues.lookupOrd(expectedGlobalOrd)); + } + } + + private static SortedSetDocValues asOrds(BytesRef[] values) { + return new AbstractSortedSetDocValues() { + + @Override + public boolean advanceExact(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long nextOrd() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef lookupOrd(long ord) throws IOException { + return values[Math.toIntExact(ord)]; + } + + @Override + public long getValueCount() { + return values.length; + } + }; + } + public void testMissingLongs() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final int[][] values = new int[numDocs][]; diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index b41ba7a85f7..f0cad042158 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -56,7 +56,6 @@ import java.util.function.Function; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -635,6 +634,18 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"), equalTo("fox ate rabbit x y z")); + + // Source filter on a field that does not exist inside the nested document and just check that we do not fail and + // return an empty _source: + response = client().prepareSearch() + .setQuery(nestedQuery("comments", matchQuery("comments.message", "away"), ScoreMode.None) + .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, + new String[]{"comments.missing_field"}, null)))) + .get(); + assertNoFailures(response); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(1L)); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(0)); } public void testInnerHitsWithIgnoreUnmapped() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java index 45825677051..e887cb48585 100644 --- a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java +++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java @@ -237,7 +237,6 @@ public class QueryProfilerTests extends ESTestCase { @Override public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { - final Weight weight = this; return new ScorerSupplier() { @Override @@ -251,6 +250,11 @@ public class QueryProfilerTests extends ESTestCase { } }; } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } }; } } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java index ccc96c94eb7..aac22302d3b 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java @@ -21,6 +21,7 @@ package org.elasticsearch.plugins; import org.apache.lucene.util.IOUtils; import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.LoggingAwareMultiCommand; import org.elasticsearch.cli.MultiCommand; import org.elasticsearch.cli.Terminal; @@ -31,7 +32,7 @@ import java.util.Collections; /** * A cli tool for adding, removing and listing plugins for elasticsearch. */ -public class PluginCli extends MultiCommand { +public class PluginCli extends LoggingAwareMultiCommand { private final Collection commands; diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 65c9e55a365..7f6d952deca 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,7 +1,7 @@ :version: 7.0.0-alpha1 :major-version: 7.x -:lucene_version: 7.1.0 -:lucene_version_path: 7_1_0 +:lucene_version: 7.2.0-SNAPSHOT +:lucene_version_path: 7_2_0 :branch: master :jdk: 1.8.0_131 diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc index 5da117cc7cf..ca37c42fb02 100644 --- a/docs/community-clients/index.asciidoc +++ b/docs/community-clients/index.asciidoc @@ -129,7 +129,7 @@ The following project appears to be abandoned: == Lua * https://github.com/DhavalKapil/elasticsearch-lua[elasticsearch-lua]: - Lua client for elasticsearch + Lua client for Elasticsearch [[dotnet]] == .NET @@ -218,7 +218,7 @@ Also see the {client}/ruby-api/current/index.html[official Elasticsearch Ruby cl Tiny client with built-in zero-downtime migrations and ActiveRecord integration. * https://github.com/toptal/chewy[chewy]: - Chewy is ODM and wrapper for official elasticsearch client + Chewy is an ODM and wrapper for the official Elasticsearch client * https://github.com/ankane/searchkick[Searchkick]: Intelligent search made easy diff --git a/docs/groovy-api/anatomy.asciidoc b/docs/groovy-api/anatomy.asciidoc index 99e008fb6eb..2da39a05ba5 100644 --- a/docs/groovy-api/anatomy.asciidoc +++ b/docs/groovy-api/anatomy.asciidoc @@ -13,7 +13,7 @@ The first type is to simply provide the request as a Closure, which automatically gets resolved into the respective request instance (for the index API, its the `IndexRequest` class). The API returns a special future, called `GActionFuture`. This is a groovier version of -elasticsearch Java `ActionFuture` (in turn a nicer extension to Java own +Elasticsearch Java `ActionFuture` (in turn a nicer extension to Java own `Future`) which allows to register listeners (closures) on it for success and failures, as well as blocking for the response. For example: diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc index a2745f459bd..c3c89e71bc5 100644 --- a/docs/groovy-api/client.asciidoc +++ b/docs/groovy-api/client.asciidoc @@ -1,7 +1,7 @@ [[client]] == Client -Obtaining an elasticsearch Groovy `GClient` (a `GClient` is a simple +Obtaining an Elasticsearch Groovy `GClient` (a `GClient` is a simple wrapper on top of the Java `Client`) is simple. The most common way to get a client is by starting an embedded `Node` which acts as a node within the cluster. @@ -11,7 +11,7 @@ within the cluster. === Node Client A Node based client is the simplest form to get a `GClient` to start -executing operations against elasticsearch. +executing operations against Elasticsearch. [source,groovy] -------------------------------------------------- @@ -29,7 +29,7 @@ GClient client = node.client(); node.close(); -------------------------------------------------- -Since elasticsearch allows to configure it using JSON based settings, +Since Elasticsearch allows to configure it using JSON based settings, the configuration itself can be done using a closure that represent the JSON: diff --git a/docs/groovy-api/index.asciidoc b/docs/groovy-api/index.asciidoc index dd819c07c1e..e1bb81856f1 100644 --- a/docs/groovy-api/index.asciidoc +++ b/docs/groovy-api/index.asciidoc @@ -6,7 +6,7 @@ include::../Versions.asciidoc[] == Preface This section describes the http://groovy-lang.org/[Groovy] API -elasticsearch provides. All elasticsearch APIs are executed using a +Elasticsearch provides. All Elasticsearch APIs are executed using a <>, and are completely asynchronous in nature (they either accept a listener, or return a future). diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc index 83889bda49a..d5893ffce95 100644 --- a/docs/java-api/client.asciidoc +++ b/docs/java-api/client.asciidoc @@ -8,7 +8,7 @@ You can use the *Java client* in multiple ways: existing cluster * Perform administrative tasks on a running cluster -Obtaining an elasticsearch `Client` is simple. The most common way to +Obtaining an Elasticsearch `Client` is simple. The most common way to get a client is by creating a <> that connects to a cluster. @@ -69,7 +69,7 @@ After this, the client will call the internal cluster state API on those nodes to discover available data nodes. The internal node list of the client will be replaced with those data nodes only. This list is refreshed every five seconds by default. Note that the IP addresses the sniffer connects to are the ones declared as the 'publish' -address in those node's elasticsearch config. +address in those node's Elasticsearch config. Keep in mind that the list might possibly not include the original node it connected to if that node is not a data node. If, for instance, you initially connect to a diff --git a/docs/java-api/docs/bulk.asciidoc b/docs/java-api/docs/bulk.asciidoc index 07849164a68..57f55b5171a 100644 --- a/docs/java-api/docs/bulk.asciidoc +++ b/docs/java-api/docs/bulk.asciidoc @@ -78,7 +78,7 @@ BulkProcessor bulkProcessor = BulkProcessor.builder( BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) <9> .build(); -------------------------------------------------- -<1> Add your elasticsearch client +<1> Add your Elasticsearch client <2> This method is called just before bulk is executed. You can for example see the numberOfActions with `request.numberOfActions()` <3> This method is called after bulk execution. You can for example check if there was some failing requests @@ -138,7 +138,7 @@ all bulk requests to complete then returns `true`, if the specified waiting time [[java-docs-bulk-processor-tests]] ==== Using Bulk Processor in tests -If you are running tests with elasticsearch and are using the `BulkProcessor` to populate your dataset +If you are running tests with Elasticsearch and are using the `BulkProcessor` to populate your dataset you should better set the number of concurrent requests to `0` so the flush operation of the bulk will be executed in a synchronous manner: diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc index 10430adc9a6..75e97f8a45a 100644 --- a/docs/java-api/index.asciidoc +++ b/docs/java-api/index.asciidoc @@ -5,8 +5,8 @@ include::../Versions.asciidoc[] [preface] == Preface -This section describes the Java API that elasticsearch provides. All -elasticsearch operations are executed using a +This section describes the Java API that Elasticsearch provides. All +Elasticsearch operations are executed using a <> object. All operations are completely asynchronous in nature (either accepts a listener, or returns a future). diff --git a/docs/java-api/indexed-scripts.asciidoc b/docs/java-api/indexed-scripts.asciidoc index a1259649a77..f1877bba1f8 100644 --- a/docs/java-api/indexed-scripts.asciidoc +++ b/docs/java-api/indexed-scripts.asciidoc @@ -2,7 +2,7 @@ == Indexed Scripts API The indexed script API allows one to interact with scripts and templates -stored in an elasticsearch index. It can be used to create, update, get, +stored in an Elasticsearch index. It can be used to create, update, get, and delete indexed scripts and templates. [source,java] diff --git a/docs/java-rest/low-level/sniffer.asciidoc b/docs/java-rest/low-level/sniffer.asciidoc index 4af61b5d2f3..df772643bf4 100644 --- a/docs/java-rest/low-level/sniffer.asciidoc +++ b/docs/java-rest/low-level/sniffer.asciidoc @@ -16,10 +16,10 @@ The javadoc for the REST client sniffer can be found at {rest-client-sniffer-jav === Maven Repository The REST client sniffer is subject to the same release cycle as -elasticsearch. Replace the version with the desired sniffer version, first +Elasticsearch. Replace the version with the desired sniffer version, first released with `5.0.0-alpha4`. There is no relation between the sniffer version -and the elasticsearch version that the client can communicate with. Sniffer -supports fetching the nodes list from elasticsearch 2.x and onwards. +and the Elasticsearch version that the client can communicate with. Sniffer +supports fetching the nodes list from Elasticsearch 2.x and onwards. ==== Maven configuration diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc index d39fab38dda..6e25c506acb 100644 --- a/docs/java-rest/low-level/usage.asciidoc +++ b/docs/java-rest/low-level/usage.asciidoc @@ -17,10 +17,10 @@ http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Ma Central]. The minimum Java version required is `1.7`. The low-level REST client is subject to the same release cycle as -elasticsearch. Replace the version with the desired client version, first +Elasticsearch. Replace the version with the desired client version, first released with `5.0.0-alpha4`. There is no relation between the client version -and the elasticsearch version that the client can communicate with. The -low-level REST client is compatible with all elasticsearch versions. +and the Elasticsearch version that the client can communicate with. The +low-level REST client is compatible with all Elasticsearch versions. [[java-rest-low-usage-maven-maven]] ==== Maven configuration diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc index 3b91fb76f66..8dc06d1433a 100644 --- a/docs/plugins/authors.asciidoc +++ b/docs/plugins/authors.asciidoc @@ -46,7 +46,7 @@ can fill in the necessary values in the `build.gradle` file for your plugin. Use the system property `java.specification.version`. Version string must be a sequence of nonnegative decimal integers separated by "."'s and may have leading zeros. -|`elasticsearch.version` |String | version of elasticsearch compiled against. +|`elasticsearch.version` |String | version of Elasticsearch compiled against. |======================================================================= @@ -57,7 +57,7 @@ If you need other resources, package them into a resources jar. .Plugin release lifecycle ============================================== -You will have to release a new version of the plugin for each new elasticsearch release. +You will have to release a new version of the plugin for each new Elasticsearch release. This version is checked when the plugin is loaded so Elasticsearch will refuse to start in the presence of plugins with the incorrect `elasticsearch.version`. @@ -86,7 +86,7 @@ with a large warning, and they will have to confirm them when installing the plugin interactively. So if possible, it is best to avoid requesting any spurious permissions! -If you are using the elasticsearch Gradle build system, place this file in +If you are using the Elasticsearch Gradle build system, place this file in `src/main/plugin-metadata` and it will be applied during unit tests as well. Keep in mind that the Java security model is stack-based, and the additional diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc index 9fd1f97129b..f11b4018bf5 100644 --- a/docs/plugins/discovery-azure-classic.asciidoc +++ b/docs/plugins/discovery-azure-classic.asciidoc @@ -37,7 +37,7 @@ discovery: .Binding the network host ============================================== -The keystore file must be placed in a directory accessible by elasticsearch like the `config` directory. +The keystore file must be placed in a directory accessible by Elasticsearch like the `config` directory. It's important to define `network.host` as by default it's bound to `localhost`. @@ -95,7 +95,7 @@ The following are a list of settings that can further control the discovery: `discovery.azure.endpoint.name`:: When using `public_ip` this setting is used to identify the endpoint name - used to forward requests to elasticsearch (aka transport port name). + used to forward requests to Elasticsearch (aka transport port name). Defaults to `elasticsearch`. In Azure management console, you could define an endpoint `elasticsearch` forwarding for example requests on public IP on port 8100 to the virtual machine on port 9300. @@ -131,7 +131,7 @@ discovery: We will expose here one strategy which is to hide our Elasticsearch cluster from outside. With this strategy, only VMs behind the same virtual port can talk to each -other. That means that with this mode, you can use elasticsearch unicast +other. That means that with this mode, you can use Elasticsearch unicast discovery to build a cluster, using the Azure API to retrieve information about your nodes. @@ -177,7 +177,7 @@ cat azure-cert.pem azure-pk.pem > azure.pem.txt openssl pkcs12 -export -in azure.pem.txt -out azurekeystore.pkcs12 -name azure -noiter -nomaciter ---- -Upload the `azure-certificate.cer` file both in the elasticsearch Cloud Service (under `Manage Certificates`), +Upload the `azure-certificate.cer` file both in the Elasticsearch Cloud Service (under `Manage Certificates`), and under `Settings -> Manage Certificates`. IMPORTANT: When prompted for a password, you need to enter a non empty one. @@ -354,7 +354,7 @@ sudo dpkg -i elasticsearch-{version}.deb ---- // NOTCONSOLE -Check that elasticsearch is running: +Check that Elasticsearch is running: [source,js] ---- @@ -393,11 +393,11 @@ This command should give you a JSON result: // So much s/// but at least we test that the layout is close to matching.... [[discovery-azure-classic-long-plugin]] -===== Install elasticsearch cloud azure plugin +===== Install Elasticsearch cloud azure plugin [source,sh] ---- -# Stop elasticsearch +# Stop Elasticsearch sudo service elasticsearch stop # Install the plugin @@ -428,7 +428,7 @@ discovery: # path.data: /mnt/resource/elasticsearch/data ---- -Restart elasticsearch: +Restart Elasticsearch: [source,sh] ---- diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 60ccaba00ba..2e2bc9cf268 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -181,10 +181,10 @@ to use include the `discovery.ec2.tag.` prefix. For example, setting `discovery. filter instances with a tag key set to `stage`, and a value of `dev`. Several tags set will require all of those tags to be set for the instance to be included. -One practical use for tag filtering is when an ec2 cluster contains many nodes that are not running elasticsearch. In +One practical use for tag filtering is when an ec2 cluster contains many nodes that are not running Elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new node's discovery phase will end before it has found the cluster (which will result in it declaring itself master of a new cluster with the same -name - highly undesirable). Tagging elasticsearch ec2 nodes and then filtering by that tag will resolve this issue. +name - highly undesirable). Tagging Elasticsearch ec2 nodes and then filtering by that tag will resolve this issue. [[discovery-ec2-attributes]] ===== Automatic Node Attributes diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index 4b31158b694..c194434414e 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -201,7 +201,7 @@ sudo dpkg -i elasticsearch-2.0.0.deb -------------------------------------------------- [[discovery-gce-usage-long-install-plugin]] -===== Install elasticsearch discovery gce plugin +===== Install Elasticsearch discovery gce plugin Install the plugin: @@ -231,7 +231,7 @@ discovery: -------------------------------------------------- -Start elasticsearch: +Start Elasticsearch: [source,sh] -------------------------------------------------- @@ -354,9 +354,9 @@ For example, setting `discovery.gce.tags` to `dev` will only filter instances ha set will require all of those tags to be set for the instance to be included. One practical use for tag filtering is when an GCE cluster contains many nodes that are not running -elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new +Elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new node's discovery phase will end before it has found the cluster (which will result in it declaring itself master of a -new cluster with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that +new cluster with the same name - highly undesirable). Adding tag on Elasticsearch GCE nodes and then filtering by that tag will resolve this issue. Add your tag when building the new instance: @@ -385,8 +385,8 @@ discovery: [[discovery-gce-usage-port]] ==== Changing default transport port -By default, elasticsearch GCE plugin assumes that you run elasticsearch on 9300 default port. -But you can specify the port value elasticsearch is meant to use using google compute engine metadata `es_port`: +By default, Elasticsearch GCE plugin assumes that you run Elasticsearch on 9300 default port. +But you can specify the port value Elasticsearch is meant to use using google compute engine metadata `es_port`: [[discovery-gce-usage-port-create]] ===== When creating instance diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc index 9ece2febde1..162988fe3fc 100644 --- a/docs/plugins/integrations.asciidoc +++ b/docs/plugins/integrations.asciidoc @@ -60,10 +60,10 @@ releases 2.0 and later do not support rivers. The Java Database Connection (JDBC) importer allows to fetch data from JDBC sources for indexing into Elasticsearch (by Jörg Prante) * https://github.com/reachkrishnaraj/kafka-elasticsearch-standalone-consumer/tree/branch2.0[Kafka Standalone Consumer(Indexer)]: - Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into ElasticSearch. Flexible and scalable. More documentation in above GitHub repo's Wiki.(Please use branch 2.0)! + Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into Elasticsearch. Flexible and scalable. More documentation in above GitHub repo's Wiki.(Please use branch 2.0)! * https://github.com/ozlerhakan/mongolastic[Mongolastic]: - A tool that clones data from ElasticSearch to MongoDB and vice versa + A tool that clones data from Elasticsearch to MongoDB and vice versa * https://github.com/Aconex/scrutineer[Scrutineer]: A high performance consistency checker to compare what you've indexed @@ -106,7 +106,7 @@ releases 2.0 and later do not support rivers. indexing in Elasticsearch. * https://camel.apache.org/elasticsearch.html[Apache Camel Integration]: - An Apache camel component to integrate elasticsearch + An Apache camel component to integrate Elasticsearch * https://metacpan.org/release/Catmandu-Store-ElasticSearch[Catmanadu]: An Elasticsearch backend for the Catmandu framework. @@ -164,7 +164,7 @@ releases 2.0 and later do not support rivers. Nagios. * https://github.com/radu-gheorghe/check-es[check-es]: - Nagios/Shinken plugins for checking on elasticsearch + Nagios/Shinken plugins for checking on Elasticsearch * https://github.com/mattweber/es2graphite[es2graphite]: Send cluster and indices stats and status to Graphite for monitoring and graphing. @@ -206,7 +206,7 @@ These projects appear to have been abandoned: Daikon Elasticsearch CLI * https://github.com/fullscale/dangle[dangle]: - A set of AngularJS directives that provide common visualizations for elasticsearch based on + A set of AngularJS directives that provide common visualizations for Elasticsearch based on D3. * https://github.com/OlegKunitsyn/eslogd[eslogd]: Linux daemon that replicates events to a central Elasticsearch server in realtime diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 583217324ed..9f80e4ab91c 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -34,7 +34,7 @@ bin/elasticsearch-keystore add azure.client.secondary.key `default` is the default account name which will be used by a repository unless you set an explicit one. You can set the client side timeout to use when making any single request. It can be defined globally, per account or both. -It's not set by default which means that elasticsearch is using the +It's not set by default which means that Elasticsearch is using the http://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default value] set by the azure client (known as 5 minutes). diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc index 90c6e8f0ba4..d7626d69090 100644 --- a/docs/plugins/repository-gcs.asciidoc +++ b/docs/plugins/repository-gcs.asciidoc @@ -42,7 +42,7 @@ The bucket should now be created. The plugin supports two authentication modes: * The built-in <>. This mode is -recommended if your elasticsearch node is running on a Compute Engine virtual machine. +recommended if your Elasticsearch node is running on a Compute Engine virtual machine. * Specifying <> credentials. @@ -61,7 +61,7 @@ instance details at the section "Cloud API access scopes". [[repository-gcs-using-service-account]] ===== Using a Service Account -If your elasticsearch node is not running on Compute Engine, or if you don't want to use Google's +If your Elasticsearch node is not running on Compute Engine, or if you don't want to use Google's built-in authentication mechanism, you can authenticate on the Storage service using a https://cloud.google.com/iam/docs/overview#service_account[Service Account] file. diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc index 933f6f69f62..ffd5ecebc25 100644 --- a/docs/plugins/repository-hdfs.asciidoc +++ b/docs/plugins/repository-hdfs.asciidoc @@ -95,7 +95,7 @@ methods are supported by the plugin: `simple`:: Also means "no security" and is enabled by default. Uses information from underlying operating system account - running elasticsearch to inform Hadoop of the name of the current user. Hadoop makes no attempts to verify this + running Elasticsearch to inform Hadoop of the name of the current user. Hadoop makes no attempts to verify this information. `kerberos`:: diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 565c94f5a7d..eb0828e96c6 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -284,7 +284,7 @@ You may further restrict the permissions by specifying a prefix within the bucke // NOTCONSOLE The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository -registration will fail. If you want elasticsearch to create the bucket instead, you can add the permission to create a +registration will fail. If you want Elasticsearch to create the bucket instead, you can add the permission to create a specific bucket like this: [source,js] @@ -305,6 +305,6 @@ specific bucket like this: [float] ==== AWS VPC Bandwidth Settings -AWS instances resolve S3 endpoints to a public IP. If the elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations. +AWS instances resolve S3 endpoints to a public IP. If the Elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations. Instances residing in a public subnet in an AWS VPC will connect to S3 via the VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance. diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index ea72e07e337..2cac8729b86 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -2,7 +2,7 @@ === Date Histogram Aggregation A multi-bucket aggregation similar to the <> except it can -only be applied on date values. Since dates are represented in elasticsearch internally as long values, it is possible +only be applied on date values. Since dates are represented in Elasticsearch internally as long values, it is possible to use the normal `histogram` on dates as well, though accuracy will be compromised. The reason for this is in the fact that time based intervals are not fixed (think of leap years and on the number of days in a month). For this reason, we need special support for time based data. From a functionality perspective, this histogram supports the same features diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc index 1db54611b31..2de6a0bbb2f 100644 --- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc @@ -449,7 +449,7 @@ a consolidated review by the reducing node before the final selection. Obviously will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. -NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will +NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will override it and reset it to be equal to `size`. ===== Minimum document count diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index ba6b912780c..e768cb0b295 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -99,7 +99,7 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\.//] <1> an upper bound of the error on the document counts for each term, see <> -<2> when there are lots of unique terms, elasticsearch only returns the top terms; this number is the sum of the document counts for all buckets that are not part of the response +<2> when there are lots of unique terms, Elasticsearch only returns the top terms; this number is the sum of the document counts for all buckets that are not part of the response <3> the list of the top buckets, the meaning of `top` being defined by the <> By default, the `terms` aggregation will return the buckets for the top ten terms ordered by the `doc_count`. One can @@ -210,7 +210,7 @@ one can increase the accuracy of the returned terms and avoid the overhead of st the client. -NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will +NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will override it and reset it to be equal to `size`. @@ -740,7 +740,7 @@ expire then we may be missing accounts of interest and have set our numbers too * increase the `size` parameter to return more results per partition (could be heavy on memory) or * increase the `num_partitions` to consider less accounts per request (could increase overall processing time as we need to make more requests) -Ultimately this is a balancing act between managing the elasticsearch resources required to process a single request and the volume +Ultimately this is a balancing act between managing the Elasticsearch resources required to process a single request and the volume of requests that the client application must issue to complete a task. ==== Multi-field terms aggregation diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index c861a0ef9b6..3e06c6f347f 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -161,7 +161,7 @@ counting millions of items. On string fields that have a high cardinality, it might be faster to store the hash of your field values in your index and then run the cardinality aggregation on this field. This can either be done by providing hash values from client-side -or by letting elasticsearch compute hash values for you by using the +or by letting Elasticsearch compute hash values for you by using the {plugins}/mapper-murmur3.html[`mapper-murmur3`] plugin. NOTE: Pre-computing hashes is usually only useful on very large and/or diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 7668a0df792..b2df9bacae2 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -323,6 +323,9 @@ Top hits response snippet with a nested hit, which resides in the first slot of "max_score": 0.2876821, "hits": [ { + "_index": "sales", + "_type" : "product", + "_id": "1", "_nested": { "field": "comments", <1> "offset": 0 <2> diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 472c48e5232..3d8cba18e05 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -3,7 +3,7 @@ [partintro] -- -The *elasticsearch* REST APIs are exposed using <>. +The *Elasticsearch* REST APIs are exposed using <>. The conventions listed in this chapter can be applied throughout the REST API, unless otherwise specified. @@ -228,7 +228,7 @@ Some examples are: === Response Filtering All REST APIs accept a `filter_path` parameter that can be used to reduce -the response returned by elasticsearch. This parameter takes a comma +the response returned by Elasticsearch. This parameter takes a comma separated list of filters expressed with the dot notation: [source,js] @@ -360,7 +360,7 @@ Responds: -------------------------------------------------- // TESTRESPONSE -Note that elasticsearch sometimes returns directly the raw value of a field, +Note that Elasticsearch sometimes returns directly the raw value of a field, like the `_source` field. If you want to filter `_source` fields, you should consider combining the already existing `_source` parameter (see <> for more details) with the `filter_path` diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index 746d0b4bb58..3a50a836d0f 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -32,7 +32,7 @@ All these exposed metrics come directly from Lucene APIs. 1. As the number of documents and deleted documents shown in this are at the lucene level, it includes all the hidden documents (e.g. from nested documents) as well. -2. To get actual count of documents at the elasticsearch level, the recommended way +2. To get actual count of documents at the Elasticsearch level, the recommended way is to use either the <> or the <> [float] diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 0ac58622d51..74d0ee157fd 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -133,7 +133,7 @@ version number is equal to zero. A nice side effect is that there is no need to maintain strict ordering of async indexing operations executed as a result of changes to a source database, as long as version numbers from the source database are used. -Even the simple case of updating the elasticsearch index using data from +Even the simple case of updating the Elasticsearch index using data from a database is simplified if external versioning is used, as only the latest version will be used if the index operations are out of order for whatever reason. @@ -355,7 +355,7 @@ and isn't able to compare it against the new source. There isn't a hard and fast rule about when noop updates aren't acceptable. It's a combination of lots of factors like how frequently your data source sends updates that are actually noops and how many queries per second -elasticsearch runs on the shard with receiving the updates. +Elasticsearch runs on the shard with receiving the updates. [float] [[timeout]] diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index e1876327504..77ff6162dcf 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -1030,7 +1030,7 @@ PUT metricbeat-2016.05.31/beat/1?refresh ---------------------------------------------------------------- // CONSOLE -The new template for the `metricbeat-*` indices is already loaded into elasticsearch +The new template for the `metricbeat-*` indices is already loaded into Elasticsearch but it applies only to the newly created indices. Painless can be used to reindex the existing documents and apply the new template. diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc index 3559cb8985b..0012beebdca 100644 --- a/docs/reference/glossary.asciidoc +++ b/docs/reference/glossary.asciidoc @@ -16,7 +16,7 @@ terms stored in the index. + It is this process of analysis (both at index time and at search time) - that allows elasticsearch to perform full text queries. + that allows Elasticsearch to perform full text queries. + Also see <> and <>. @@ -29,7 +29,7 @@ [[glossary-document]] document :: - A document is a JSON document which is stored in elasticsearch. It is + A document is a JSON document which is stored in Elasticsearch. It is like a row in a table in a relational database. Each document is stored in an <> and has a <> and an <>. @@ -82,7 +82,7 @@ [[glossary-node]] node :: - A node is a running instance of elasticsearch which belongs to a + A node is a running instance of Elasticsearch which belongs to a <>. Multiple nodes can be started on a single server for testing purposes, but usually you should have one node per server. @@ -136,7 +136,7 @@ [[glossary-shard]] shard :: A shard is a single Lucene instance. It is a low-level “worker” unit - which is managed automatically by elasticsearch. An index is a logical + which is managed automatically by Elasticsearch. An index is a logical namespace which points to <> and <> shards. + @@ -159,7 +159,7 @@ [[glossary-term]] term :: - A term is an exact value that is indexed in elasticsearch. The terms + A term is an exact value that is indexed in Elasticsearch. The terms `foo`, `Foo`, `FOO` are NOT equivalent. Terms (i.e. exact values) can be searched for using _term_ queries. + See also <> and <>. diff --git a/docs/reference/how-to/disk-usage.asciidoc b/docs/reference/how-to/disk-usage.asciidoc index aa39f28b2fd..62218264bc9 100644 --- a/docs/reference/how-to/disk-usage.asciidoc +++ b/docs/reference/how-to/disk-usage.asciidoc @@ -4,7 +4,7 @@ [float] === Disable the features you do not need -By default elasticsearch indexes and adds doc values to most fields so that they +By default Elasticsearch indexes and adds doc values to most fields so that they can be searched and aggregated out of the box. For instance if you have a numeric field called `foo` that you need to run histograms on but that you never need to filter on, you can safely disable indexing on this field in your @@ -30,7 +30,7 @@ PUT index <> fields store normalization factors in the index in order to be able to score documents. If you only need matching capabilities on a `text` -field but do not care about the produced scores, you can configure elasticsearch +field but do not care about the produced scores, you can configure Elasticsearch to not write norms to the index: [source,js] @@ -54,7 +54,7 @@ PUT index <> fields also store frequencies and positions in the index by default. Frequencies are used to compute scores and positions are used to run phrase queries. If you do not need to run phrase queries, you can tell -elasticsearch to not index positions: +Elasticsearch to not index positions: [source,js] -------------------------------------------------- @@ -75,7 +75,7 @@ PUT index // CONSOLE Furthermore if you do not care about scoring either, you can configure -elasticsearch to just index matching documents for every term. You will +Elasticsearch to just index matching documents for every term. You will still be able to search on this field, but phrase queries will raise errors and scoring will assume that terms appear only once in every document. diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc index db7479f9f7d..3661d8ce07d 100644 --- a/docs/reference/how-to/indexing-speed.asciidoc +++ b/docs/reference/how-to/indexing-speed.asciidoc @@ -64,13 +64,13 @@ process by <>. === Give memory to the filesystem cache The filesystem cache will be used in order to buffer I/O operations. You should -make sure to give at least half the memory of the machine running elasticsearch +make sure to give at least half the memory of the machine running Elasticsearch to the filesystem cache. [float] === Use auto-generated ids -When indexing a document that has an explicit id, elasticsearch needs to check +When indexing a document that has an explicit id, Elasticsearch needs to check whether a document with the same id already exists within the same shard, which is a costly operation and gets even more costly as the index grows. By using auto-generated ids, Elasticsearch can skip this check, which makes indexing diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 60168ab856d..db84acd3516 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -6,7 +6,7 @@ Elasticsearch heavily relies on the filesystem cache in order to make search fast. In general, you should make sure that at least half the available memory -goes to the filesystem cache so that elasticsearch can keep hot regions of the +goes to the filesystem cache so that Elasticsearch can keep hot regions of the index in physical memory. [float] @@ -275,8 +275,8 @@ merging to the background merge process. Global ordinals are a data-structure that is used in order to run <> aggregations on <> fields. They are loaded lazily in memory because -elasticsearch does not know which fields will be used in `terms` aggregations -and which fields won't. You can tell elasticsearch to load global ordinals +Elasticsearch does not know which fields will be used in `terms` aggregations +and which fields won't. You can tell Elasticsearch to load global ordinals eagerly at refresh-time by configuring mappings as described below: [source,js] @@ -300,7 +300,7 @@ PUT index [float] === Warm up the filesystem cache -If the machine running elasticsearch is restarted, the filesystem cache will be +If the machine running Elasticsearch is restarted, the filesystem cache will be empty, so it will take some time before the operating system loads hot regions of the index into memory so that search operations are fast. You can explicitly tell the operating system which files should be loaded into memory eagerly diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index bf93f62847f..999bd948a15 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -107,11 +107,22 @@ specific index module: Set to a dash delimited lower and upper bound (e.g. `0-5`) or use `all` for the upper bound (e.g. `0-all`). Defaults to `false` (i.e. disabled). +`index.search.idle.after`:: + How long a shard can not receive a search or get request until it's considered + search idle. (default is `30s`) + `index.refresh_interval`:: How often to perform a refresh operation, which makes recent changes to the - index visible to search. Defaults to `1s`. Can be set to `-1` to disable - refresh. + index visible to search. Defaults to `1s`. Can be set to `-1` to disable + refresh. If this setting is not explicitly set, shards that haven't seen + search traffic for at least `index.search.idle.after` seconds will not receive + background refreshes until they receive a search request. Searches that hit an + idle shard where a refresh is pending will wait for the next background + refresh (within `1s`). This behavior aims to automatically optimize bulk + indexing in the default case when no searches are performed. In order to opt + out of this behavior an explicit value of `1s` should set as the refresh + interval. `index.max_result_window`:: @@ -169,7 +180,9 @@ specific index module: `index.blocks.write`:: - Set to `true` to disable write operations against the index. +   Set to `true` to disable data write operations against the index. Unlike `read_only', +   this setting does not affect metadata. For instance, you can close an index with a `write` +   block, but not an index with a `read_only` block. `index.blocks.metadata`:: diff --git a/docs/reference/index-modules/index-sorting.asciidoc b/docs/reference/index-modules/index-sorting.asciidoc index 8aede7492df..3947b2501c2 100644 --- a/docs/reference/index-modules/index-sorting.asciidoc +++ b/docs/reference/index-modules/index-sorting.asciidoc @@ -3,7 +3,7 @@ beta[] -When creating a new index in elasticsearch it is possible to configure how the Segments +When creating a new index in Elasticsearch it is possible to configure how the Segments inside each Shard will be sorted. By default Lucene does not apply any sort. The `index.sort.*` settings define which fields should be used to sort the documents inside each Segment. @@ -112,7 +112,7 @@ before activating this feature. [[early-terminate]] === Early termination of search request -By default in elasticsearch a search request must visit every document that match a query to +By default in Elasticsearch a search request must visit every document that match a query to retrieve the top documents sorted by a specified sort. Though when the index sort and the search sort are the same it is possible to limit the number of documents that should be visited per segment to retrieve the N top ranked documents globally. diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index 7e5260f95d4..97db09ba656 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -1,7 +1,7 @@ [[index-modules-merge]] == Merge -A shard in elasticsearch is a Lucene index, and a Lucene index is broken down +A shard in Elasticsearch is a Lucene index, and a Lucene index is broken down into segments. Segments are internal storage elements in the index where the index data is stored, and are immutable. Smaller segments are periodically merged into larger segments to keep the index size at bay and to expunge diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index 27b5b751233..e657affb607 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -8,7 +8,7 @@ The store module allows you to control how index data is stored and accessed on === File system storage types There are different file system implementations or _storage types_. By default, -elasticsearch will pick the best implementation based on the operating +Elasticsearch will pick the best implementation based on the operating environment. This can be overridden for all indices by adding this to the @@ -76,7 +76,7 @@ compatibility. NOTE: This is an expert setting, the details of which may change in the future. -By default, elasticsearch completely relies on the operating system file system +By default, Elasticsearch completely relies on the operating system file system cache for caching I/O operations. It is possible to set `index.store.preload` in order to tell the operating system to load the content of hot index files into memory upon opening. This setting accept a comma-separated list of @@ -115,7 +115,7 @@ The default value is the empty array, which means that nothing will be loaded into the file-system cache eagerly. For indices that are actively searched, you might want to set it to `["nvd", "dvd"]`, which will cause norms and doc values to be loaded eagerly into physical memory. These are the two first -extensions to look at since elasticsearch performs random access on them. +extensions to look at since Elasticsearch performs random access on them. A wildcard can be used in order to indicate that all files should be preloaded: `index.store.preload: ["*"]`. Note however that it is generally not useful to diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index c8a24ba4614..90d4ae1de56 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -257,9 +257,9 @@ Here are some examples of potentially useful dynamic templates: ===== Structured search -By default elasticsearch will map string fields as a `text` field with a sub +By default Elasticsearch will map string fields as a `text` field with a sub `keyword` field. However if you are only indexing structured content and not -interested in full text search, you can make elasticsearch map your fields +interested in full text search, you can make Elasticsearch map your fields only as `keyword`s. Note that this means that in order to search those fields, you will have to search on the exact same value that was indexed. @@ -290,7 +290,7 @@ PUT my_index On the contrary to the previous example, if the only thing that you care about on your string fields is full-text search, and if you don't plan on running aggregations, sorting or exact search on your string fields, you could tell -elasticsearch to map it only as a text field (which was the default behaviour +Elasticsearch to map it only as a text field (which was the default behaviour before 5.0): [source,js] @@ -357,7 +357,7 @@ remove it as described in the previous section. ===== Time-series -When doing time series analysis with elasticsearch, it is common to have many +When doing time series analysis with Elasticsearch, it is common to have many numeric fields that you will often aggregate on but never filter on. In such a case, you could disable indexing on those fields to save disk space and also maybe gain some indexing speed: diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 56b67d9fa56..757c958fb16 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -8,10 +8,10 @@ The following numeric types are supported: `integer`:: A signed 32-bit integer with a minimum value of +-2^31^+ and a maximum value of +2^31^-1+. `short`:: A signed 16-bit integer with a minimum value of +-32,768+ and a maximum value of +32,767+. `byte`:: A signed 8-bit integer with a minimum value of +-128+ and a maximum value of +127+. -`double`:: A double-precision 64-bit IEEE 754 floating point. -`float`:: A single-precision 32-bit IEEE 754 floating point. -`half_float`:: A half-precision 16-bit IEEE 754 floating point. -`scaled_float`:: A floating point that is backed by a `long` and a fixed scaling factor. +`double`:: A double-precision 64-bit IEEE 754 floating point number. +`float`:: A single-precision 32-bit IEEE 754 floating point number. +`half_float`:: A half-precision 16-bit IEEE 754 floating point number. +`scaled_float`:: A floating point number that is backed by a `long`, scaled by a fixed `double` scaling factor. Below is an example of configuring a mapping with numeric fields: @@ -49,15 +49,15 @@ bound is `+0.0` then `-0.0` will not match. As far as integer types (`byte`, `short`, `integer` and `long`) are concerned, you should pick the smallest type which is enough for your use-case. This will -help indexing and searching be more efficient. Note however that given that -storage is optimized based on the actual values that are stored, picking one -type over another one will have no impact on storage requirements. +help indexing and searching be more efficient. Note however that storage is +optimized based on the actual values that are stored, so picking one type over +another one will have no impact on storage requirements. For floating-point types, it is often more efficient to store floating-point data into an integer using a scaling factor, which is what the `scaled_float` type does under the hood. For instance, a `price` field could be stored in a `scaled_float` with a `scaling_factor` of +100+. All APIs would work as if -the field was stored as a double, but under the hood elasticsearch would be +the field was stored as a double, but under the hood Elasticsearch would be working with the number of cents, +price*100+, which is an integer. This is mostly helpful to save disk space since integers are way easier to compress than floating points. `scaled_float` is also fine to use in order to trade diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc index 75a5131f8fb..7d84077ab86 100644 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ b/docs/reference/migration/migrate_7_0.asciidoc @@ -35,10 +35,10 @@ way to reindex old indices is to use the `reindex` API. include::migrate_7_0/aggregations.asciidoc[] +include::migrate_7_0/analysis.asciidoc[] include::migrate_7_0/cluster.asciidoc[] include::migrate_7_0/indices.asciidoc[] include::migrate_7_0/mappings.asciidoc[] include::migrate_7_0/search.asciidoc[] include::migrate_7_0/plugins.asciidoc[] include::migrate_7_0/api.asciidoc[] - diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc index 92f56a2ddbb..16e437b4156 100644 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ b/docs/reference/migration/migrate_7_0/indices.asciidoc @@ -44,4 +44,14 @@ Indices created with version `7.0.0` onwards will have an automatic `index.numbe value set. This might change how documents are distributed across shards depending on how many shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the `index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time. -Note: if the number of routing shards equals the number of shards `_split` operations are not supported. \ No newline at end of file +Note: if the number of routing shards equals the number of shards `_split` operations are not supported. + +==== Skipped background refresh on search idle shards. + +Shards belonging to an index that does not have an explicit +`index.refresh_interval` configured will no longer refresh in the background +once the shard becomes "search idle", ie the shard hasn't seen any search +traffic for `index.search.idle.after` seconds (defaults to `30s`). Searches +that access a search idle shard will be "parked" until the next refresh +happens. Indexing requests with `wait_for_refresh` will also trigger +a background refresh. diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc index e68350515ea..0cce897f115 100644 --- a/docs/reference/modules/discovery/zen.asciidoc +++ b/docs/reference/modules/discovery/zen.asciidoc @@ -1,7 +1,7 @@ [[modules-discovery-zen]] === Zen Discovery -The zen discovery is the built in discovery module for elasticsearch and +The zen discovery is the built in discovery module for Elasticsearch and the default. It provides unicast discovery, but can be extended to support cloud environments and other forms of discovery. diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index 065c91349c2..a83270ec2aa 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -1,7 +1,7 @@ [[modules-http]] == HTTP -The http module allows to expose *elasticsearch* APIs +The http module allows to expose *Elasticsearch* APIs over HTTP. The http mechanism is completely asynchronous in nature, meaning that @@ -74,7 +74,7 @@ allowed. If you prepend and append a `/` to the value, this will be treated as a regular expression, allowing you to support HTTP and HTTPs. for example using `/https?:\/\/localhost(:[0-9]+)?/` would return the request header appropriately in both cases. `*` is a valid value but is -considered a *security risk* as your elasticsearch instance is open to cross origin +considered a *security risk* as your Elasticsearch instance is open to cross origin requests from *anywhere*. |`http.cors.max-age` |Browsers send a "preflight" OPTIONS-request to diff --git a/docs/reference/modules/memcached.asciidoc b/docs/reference/modules/memcached.asciidoc index b3845c74a3c..508d328671b 100644 --- a/docs/reference/modules/memcached.asciidoc +++ b/docs/reference/modules/memcached.asciidoc @@ -1,7 +1,7 @@ [[modules-memcached]] == memcached -The memcached module allows to expose *elasticsearch* +The memcached module allows to expose *Elasticsearch* APIs over the memcached protocol (as closely as possible). @@ -18,7 +18,7 @@ automatically detecting the correct one to use. === Mapping REST to Memcached Protocol Memcached commands are mapped to REST and handled by the same generic -REST layer in elasticsearch. Here is a list of the memcached commands +REST layer in Elasticsearch. Here is a list of the memcached commands supported: [float] diff --git a/docs/reference/modules/plugins.asciidoc b/docs/reference/modules/plugins.asciidoc index ad708e88024..0485065d2b7 100644 --- a/docs/reference/modules/plugins.asciidoc +++ b/docs/reference/modules/plugins.asciidoc @@ -4,7 +4,7 @@ [float] === Plugins -Plugins are a way to enhance the basic elasticsearch functionality in a +Plugins are a way to enhance the basic Elasticsearch functionality in a custom manner. They range from adding custom mapping types, custom analyzers (in a more built in fashion), custom script engines, custom discovery and more. diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index f5b56160049..b8883173b98 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -319,7 +319,7 @@ GET /_snapshot/my_backup/snapshot_1 // TEST[continued] This command returns basic information about the snapshot including start and end time, version of -elasticsearch that created the snapshot, the list of included indices, the current state of the +Elasticsearch that created the snapshot, the list of included indices, the current state of the snapshot and the list of failures that occurred during the snapshot. The snapshot `state` can be [horizontal] @@ -343,7 +343,7 @@ snapshot and the list of failures that occurred during the snapshot. The snapsho `INCOMPATIBLE`:: - The snapshot was created with an old version of elasticsearch and therefore is incompatible with + The snapshot was created with an old version of Elasticsearch and therefore is incompatible with the current version of the cluster. diff --git a/docs/reference/modules/thrift.asciidoc b/docs/reference/modules/thrift.asciidoc index fc0d08cdd7b..1ea3f818126 100644 --- a/docs/reference/modules/thrift.asciidoc +++ b/docs/reference/modules/thrift.asciidoc @@ -2,7 +2,7 @@ == Thrift The https://thrift.apache.org/[thrift] transport module allows to expose the REST interface of -elasticsearch using thrift. Thrift should provide better performance +Elasticsearch using thrift. Thrift should provide better performance over http. Since thrift provides both the wire protocol and the transport, it should make using Elasticsearch more efficient (though it has limited documentation). diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc index 8ed36ce8e74..50c35a4a736 100644 --- a/docs/reference/modules/transport.asciidoc +++ b/docs/reference/modules/transport.asciidoc @@ -12,7 +12,7 @@ that there is no blocking thread waiting for a response. The benefit of using asynchronous communication is first solving the http://en.wikipedia.org/wiki/C10k_problem[C10k problem], as well as being the ideal solution for scatter (broadcast) / gather operations such -as search in ElasticSearch. +as search in Elasticsearch. [float] === TCP Transport diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index c864c643c8f..b2444535153 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -401,7 +401,7 @@ Looking at the previous example: We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting" `Collector` used by Elasticsearch. The `reason` field attempts to give a plain english description of the class name. The `time_in_nanos` is similar to the time in the Query tree: a wall-clock time inclusive of all children. Similarly, `children` lists -all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by elasticsearch to detect if the current +all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by Elasticsearch to detect if the current search was cancelled and stop collecting documents as soon as it occurs. It should be noted that Collector times are **independent** from the Query times. They are calculated, combined diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index a9da7378809..fceb37418eb 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -152,6 +152,9 @@ An example of a response snippet that could be generated from the above search r "max_score": 1.0, "hits": [ { + "_index": "test", + "_type": "doc", + "_id": "1", "_nested": { "field": "comments", "offset": 1 @@ -278,6 +281,9 @@ Response not included in text but tested for completeness sake. "max_score": 1.0444683, "hits": [ { + "_index": "test", + "_type": "doc", + "_id": "1", "_nested": { "field": "comments", "offset": 1 @@ -394,6 +400,9 @@ Which would look like: "max_score": 0.6931472, "hits": [ { + "_index": "test", + "_type": "doc", + "_id": "1", "_nested": { "field": "comments", "offset": 1, @@ -505,6 +514,7 @@ An example of a response snippet that could be generated from the above search r "max_score": 1.0, "hits": [ { + "_index": "test", "_type": "doc", "_id": "2", "_score": 1.0, diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc index 08c6f7fd903..226782d9f57 100644 --- a/docs/reference/search/request/sort.asciidoc +++ b/docs/reference/search/request/sort.asciidoc @@ -143,7 +143,7 @@ favor of the options documented above. ===== Nested sorting examples In the below example `offer` is a field of type `nested`. -The nested `path` needs to be specified; otherwise, elasticsearch doesn't know on what nested level sort values need to be captured. +The nested `path` needs to be specified; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured. [source,js] -------------------------------------------------- @@ -171,7 +171,7 @@ POST /_search // CONSOLE In the below example `parent` and `child` fields are of type `nested`. -The `nested_path` needs to be specified at each level; otherwise, elasticsearch doesn't know on what nested level sort values need to be captured. +The `nested_path` needs to be specified at each level; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured. [source,js] -------------------------------------------------- diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc index cc2b9b7cf8f..625dd944109 100644 --- a/docs/reference/search/search-template.asciidoc +++ b/docs/reference/search/search-template.asciidoc @@ -26,7 +26,7 @@ For more information on how Mustache templating and what kind of templating you can do with it check out the http://mustache.github.io/mustache.5.html[online documentation of the mustache project]. -NOTE: The mustache language is implemented in elasticsearch as a sandboxed +NOTE: The mustache language is implemented in Elasticsearch as a sandboxed scripting language, hence it obeys settings that may be used to enable or disable scripts per type and context as described in the <> diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 816cf54c529..77ae8a47dd9 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -60,8 +60,8 @@ GET /_search?q=tag:wow // CONSOLE // TEST[setup:twitter] -By default elasticsearch doesn't reject any search requests based on the number -of shards the request hits. While elasticsearch will optimize the search execution +By default Elasticsearch doesn't reject any search requests based on the number +of shards the request hits. While Elasticsearch will optimize the search execution on the coordinating node a large number of shards can have a significant impact CPU and memory wise. It is usually a better idea to organize data in such a way that there are fewer larger shards. In case you would like to configure a soft diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 3fd5b6053fa..58e9867519d 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -227,3 +227,9 @@ have issues that can lead to index corruption when the G1GC collector is enabled. The versions impacted are those earlier than the version of HotSpot that shipped with JDK 8u40. The G1GC check detects these early versions of the HotSpot JVM. + +=== All permission check + +The all permission check ensures that the security policy used during bootstrap +does not grant the `java.security.AllPermission` to Elasticsearch. Running with +the all permission granted is equivalent to disabling the security manager. diff --git a/docs/reference/setup/install/sysconfig-file.asciidoc b/docs/reference/setup/install/sysconfig-file.asciidoc index 76a9fa22bfc..d2df5cd4e0c 100644 --- a/docs/reference/setup/install/sysconfig-file.asciidoc +++ b/docs/reference/setup/install/sysconfig-file.asciidoc @@ -19,7 +19,7 @@ information, check the https://github.com/torvalds/linux/blob/master/Documentation/sysctl/vm.txt[linux kernel documentation] about `max_map_count`. This is set via `sysctl` before starting - elasticsearch. Defaults to `262144`. + Elasticsearch. Defaults to `262144`. `ES_PATH_CONF`:: @@ -34,7 +34,7 @@ `RESTART_ON_UPGRADE`:: Configure restart on package upgrade, defaults to `false`. This means you - will have to restart your elasticsearch instance after installing a + will have to restart your Elasticsearch instance after installing a package manually. The reason for this is to ensure, that upgrades in a cluster do not result in a continuous shard reallocation resulting in high network traffic and reducing the response times of your cluster. diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc index 2eaebe4ec28..76f7de6b9c1 100644 --- a/docs/reference/setup/secure-settings.asciidoc +++ b/docs/reference/setup/secure-settings.asciidoc @@ -2,11 +2,11 @@ === Secure Settings Some settings are sensitive, and relying on filesystem permissions to protect -their values is not sufficient. For this use case, elasticsearch provides a +their values is not sufficient. For this use case, Elasticsearch provides a keystore, which may be password protected, and the `elasticsearch-keystore` tool to manage the settings in the keystore. -NOTE: All commands here should be run as the user which will run elasticsearch. +NOTE: All commands here should be run as the user which will run Elasticsearch. NOTE: Only some settings are designed to be read from the keystore. See documentation for each setting to see if it is supported as part of the keystore. diff --git a/docs/reference/setup/sysconfig/threads.asciidoc b/docs/reference/setup/sysconfig/threads.asciidoc index 1cc65b7a03b..a9a75dc872b 100644 --- a/docs/reference/setup/sysconfig/threads.asciidoc +++ b/docs/reference/setup/sysconfig/threads.asciidoc @@ -4,9 +4,9 @@ Elasticsearch uses a number of thread pools for different types of operations. It is important that it is able to create new threads whenever needed. Make sure that the number of threads that the Elasticsearch user can create is at -least 2048. +least 4096. -This can be done by setting <> as root before -starting Elasticsearch, or by setting `nproc` to `2048` in +This can be done by setting <> as root before +starting Elasticsearch, or by setting `nproc` to `4096` in <>. diff --git a/docs/reference/testing.asciidoc b/docs/reference/testing.asciidoc index fd1a460f1c5..f4e4b86acc5 100644 --- a/docs/reference/testing.asciidoc +++ b/docs/reference/testing.asciidoc @@ -3,7 +3,7 @@ [partintro] -- -This section is about utilizing elasticsearch as part of your testing infrastructure. +This section is about utilizing Elasticsearch as part of your testing infrastructure. [float] [[testing-header]] diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc index d1fe769f3c1..f5634a2558e 100644 --- a/docs/reference/testing/testing-framework.asciidoc +++ b/docs/reference/testing/testing-framework.asciidoc @@ -3,7 +3,7 @@ [[testing-intro]] -Testing is a crucial part of your application, and as information retrieval itself is already a complex topic, there should not be any additional complexity in setting up a testing infrastructure, which uses elasticsearch. This is the main reason why we decided to release an additional file to the release, which allows you to use the same testing infrastructure we do in the elasticsearch core. The testing framework allows you to setup clusters with multiple nodes in order to check if your code covers everything needed to run in a cluster. The framework prevents you from writing complex code yourself to start, stop or manage several test nodes in a cluster. In addition there is another very important feature called randomized testing, which you are getting for free as it is part of the elasticsearch infrastructure. +Testing is a crucial part of your application, and as information retrieval itself is already a complex topic, there should not be any additional complexity in setting up a testing infrastructure, which uses Elasticsearch. This is the main reason why we decided to release an additional file to the release, which allows you to use the same testing infrastructure we do in the Elasticsearch core. The testing framework allows you to setup clusters with multiple nodes in order to check if your code covers everything needed to run in a cluster. The framework prevents you from writing complex code yourself to start, stop or manage several test nodes in a cluster. In addition there is another very important feature called randomized testing, which you are getting for free as it is part of the Elasticsearch infrastructure. @@ -16,9 +16,9 @@ All of the tests are run using a custom junit runner, the `RandomizedRunner` pro [[using-elasticsearch-test-classes]] -=== Using the elasticsearch test classes +=== Using the Elasticsearch test classes -First, you need to include the testing dependency in your project, along with the elasticsearch dependency you have already added. If you use maven and its `pom.xml` file, it looks like this +First, you need to include the testing dependency in your project, along with the Elasticsearch dependency you have already added. If you use maven and its `pom.xml` file, it looks like this [source,xml] -------------------------------------------------- @@ -50,7 +50,7 @@ We provide a few classes that you can inherit from in your own test classes whic [[unit-tests]] === unit tests -If your test is a well isolated unit test which doesn't need a running elasticsearch cluster, you can use the `ESTestCase`. If you are testing lucene features, use `ESTestCase` and if you are testing concrete token streams, use the `ESTokenStreamTestCase` class. Those specific classes execute additional checks which ensure that no resources leaks are happening, after the test has run. +If your test is a well isolated unit test which doesn't need a running Elasticsearch cluster, you can use the `ESTestCase`. If you are testing lucene features, use `ESTestCase` and if you are testing concrete token streams, use the `ESTokenStreamTestCase` class. Those specific classes execute additional checks which ensure that no resources leaks are happening, after the test has run. [[integration-tests]] @@ -58,7 +58,7 @@ If your test is a well isolated unit test which doesn't need a running elasticse These kind of tests require firing up a whole cluster of nodes, before the tests can actually be run. Compared to unit tests they are obviously way more time consuming, but the test infrastructure tries to minimize the time cost by only restarting the whole cluster, if this is configured explicitly. -The class your tests have to inherit from is `ESIntegTestCase`. By inheriting from this class, you will no longer need to start elasticsearch nodes manually in your test, although you might need to ensure that at least a certain number of nodes are up. The integration test behaviour can be configured heavily by specifying different system properties on test runs. See the `TESTING.asciidoc` documentation in the https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc[source repository] for more information. +The class your tests have to inherit from is `ESIntegTestCase`. By inheriting from this class, you will no longer need to start Elasticsearch nodes manually in your test, although you might need to ensure that at least a certain number of nodes are up. The integration test behaviour can be configured heavily by specifying different system properties on test runs. See the `TESTING.asciidoc` documentation in the https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc[source repository] for more information. [[number-of-shards]] @@ -100,8 +100,8 @@ The `InternalTestCluster` class is the heart of the cluster functionality in a r `stopRandomNode()`:: Stop a random node in your cluster to mimic an outage `stopCurrentMasterNode()`:: Stop the current master node to force a new election `stopRandomNonMaster()`:: Stop a random non master node to mimic an outage -`buildNode()`:: Create a new elasticsearch node -`startNode(settings)`:: Create and start a new elasticsearch node +`buildNode()`:: Create a new Elasticsearch node +`startNode(settings)`:: Create and start a new Elasticsearch node [[changing-node-settings]] @@ -165,7 +165,7 @@ data nodes will be allowed to become masters as well. [[changing-node-configuration]] ==== Changing plugins via configuration -As elasticsearch is using JUnit 4, using the `@Before` and `@After` annotations is not a problem. However you should keep in mind, that this does not have any effect in your cluster setup, as the cluster is already up and running when those methods are run. So in case you want to configure settings - like loading a plugin on node startup - before the node is actually running, you should overwrite the `nodePlugins()` method from the `ESIntegTestCase` class and return the plugin classes each node should load. +As Elasticsearch is using JUnit 4, using the `@Before` and `@After` annotations is not a problem. However you should keep in mind, that this does not have any effect in your cluster setup, as the cluster is already up and running when those methods are run. So in case you want to configure settings - like loading a plugin on node startup - before the node is actually running, you should overwrite the `nodePlugins()` method from the `ESIntegTestCase` class and return the plugin classes each node should load. [source,java] ----------------------------------------- @@ -191,7 +191,7 @@ The next step is to convert your test using static test data into a test using r * Changing your response sizes/configurable limits with each run * Changing the number of shards/replicas when creating an index -So, how can you create random data. The most important thing to know is, that you never should instantiate your own `Random` instance, but use the one provided in the `RandomizedTest`, from which all elasticsearch dependent test classes inherit from. +So, how can you create random data. The most important thing to know is, that you never should instantiate your own `Random` instance, but use the one provided in the `RandomizedTest`, from which all Elasticsearch dependent test classes inherit from. [horizontal] `getRandom()`:: Returns the random instance, which can recreated when calling the test with specific parameters @@ -221,7 +221,7 @@ If you want to debug a specific problem with a specific random seed, you can use [[assertions]] === Assertions -As many elasticsearch tests are checking for a similar output, like the amount of hits or the first hit or special highlighting, a couple of predefined assertions have been created. Those have been put into the `ElasticsearchAssertions` class. There is also a specific geo assertions in `ElasticsearchGeoAssertions`. +As many Elasticsearch tests are checking for a similar output, like the amount of hits or the first hit or special highlighting, a couple of predefined assertions have been created. Those have been put into the `ElasticsearchAssertions` class. There is also a specific geo assertions in `ElasticsearchGeoAssertions`. [horizontal] `assertHitCount()`:: Checks hit count of a search or count request diff --git a/modules/lang-expression/licenses/lucene-expressions-7.1.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.1.0.jar.sha1 deleted file mode 100644 index 29689e4e74f..00000000000 --- a/modules/lang-expression/licenses/lucene-expressions-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -714927eb1d1db641bff9aa658e7e112c368f3e6d \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..26ac38e7a0a --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +5d4b0551a5f745ddf630184b8f63d9d03b4f4003 \ No newline at end of file diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java index bd6f453083d..a0cfd721adb 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; import java.io.IOException; @@ -72,4 +73,14 @@ final class ReplaceableConstDoubleValueSource extends DoubleValuesSource { public String toString() { return getClass().getSimpleName(); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } + + @Override + public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException { + return this; + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index 4aa36ba3714..3e1c2ff2db1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -1076,9 +1076,11 @@ public final class Walker extends PainlessParserBaseVisitor { } } + FunctionReserved lambdaReserved = (FunctionReserved)reserved.pop(); + reserved.peek().addUsedVariables(lambdaReserved); + String name = nextLambda(); - return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx), - paramTypes, paramNames, statements); + return new ELambda(name, lambdaReserved, location(ctx), paramTypes, paramNames, statements); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java index 84c6145ac0c..873f109e72d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java @@ -20,7 +20,6 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Globals; @@ -213,6 +212,11 @@ public final class EAssignment extends AExpression { // If the lhs node is a def optimized node we update the actual type to remove the need for a cast. if (lhs.isDefOptimized()) { rhs.analyze(locals); + + if (rhs.actual.clazz == void.class) { + throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment.")); + } + rhs.expected = rhs.actual; lhs.updateActual(rhs.actual); // Otherwise, we must adapt the rhs type to the lhs type with a cast. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java index 89fc169704f..560acaf131e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java @@ -19,9 +19,7 @@ package org.elasticsearch.painless.node; -import java.util.Collections; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -29,6 +27,7 @@ import org.elasticsearch.painless.MethodWriter; import org.objectweb.asm.Type; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; @@ -76,6 +75,10 @@ final class PSubDefCall extends AExpression { totalCaptures += lambda.getCaptureCount(); } + if (expression.actual.clazz == void.class) { + throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "].")); + } + expression.expected = expression.actual; arguments.set(argument, expression.cast(locals)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java index c1f0c468e42..99d60c3b73f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java @@ -20,7 +20,6 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java index 6c479265cfe..6183697bbe4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -41,11 +41,13 @@ import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; /** @@ -53,11 +55,22 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; */ public final class SFunction extends AStatement { public static final class FunctionReserved implements Reserved { + private final Set usedVariables = new HashSet<>(); private int maxLoopCounter = 0; @Override public void markUsedVariable(String name) { - // Do nothing. + usedVariables.add(name); + } + + @Override + public Set getUsedVariables() { + return unmodifiableSet(usedVariables); + } + + @Override + public void addUsedVariables(FunctionReserved reserved) { + usedVariables.addAll(reserved.getUsedVariables()); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java index 882b018bc41..c1ab8398bad 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.ScriptClassInfo; import org.elasticsearch.painless.SimpleChecksAdapter; import org.elasticsearch.painless.WriterConstants; +import org.elasticsearch.painless.node.SFunction.FunctionReserved; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import org.objectweb.asm.Label; @@ -89,6 +90,8 @@ public final class SSource extends AStatement { */ public interface Reserved { void markUsedVariable(String name); + Set getUsedVariables(); + void addUsedVariables(FunctionReserved reserved); void setMaxLoopCounter(int max); int getMaxLoopCounter(); @@ -103,6 +106,16 @@ public final class SSource extends AStatement { usedVariables.add(name); } + @Override + public Set getUsedVariables() { + return unmodifiableSet(usedVariables); + } + + @Override + public void addUsedVariables(FunctionReserved reserved) { + usedVariables.addAll(reserved.getUsedVariables()); + } + @Override public void setMaxLoopCounter(int max) { maxLoopCounter = max; @@ -112,10 +125,6 @@ public final class SSource extends AStatement { public int getMaxLoopCounter() { return maxLoopCounter; } - - public Set getUsedVariables() { - return unmodifiableSet(usedVariables); - } } private final ScriptClassInfo scriptClassInfo; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java index 554da280dda..f124d088bf2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless; /** Tests for addition operator across all types */ //TODO: NaN/Inf/overflow/... public class AdditionTests extends ScriptTestCase { - + public void testBasics() throws Exception { assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;")); } @@ -195,7 +195,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(1.0+0.0, exec("return 1.0+0.0;")); assertEquals(0.0+0.0, exec("return 0.0+0.0;")); } - + public void testDef() { assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y")); assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y")); @@ -253,7 +253,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y")); assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y")); } - + public void testDefTypedLHS() { assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y")); assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y")); @@ -311,7 +311,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y")); assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y")); } - + public void testDefTypedRHS() { assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y")); assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y")); @@ -369,19 +369,19 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y")); assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y")); } - + public void testDefNulls() { expectScriptThrows(NullPointerException.class, () -> { - exec("def x = null; int y = 1; return x + y"); + exec("def x = null; int y = 1; return x + y"); }); expectScriptThrows(NullPointerException.class, () -> { - exec("int x = 1; def y = null; return x + y"); + exec("int x = 1; def y = null; return x + y"); }); expectScriptThrows(NullPointerException.class, () -> { - exec("def x = null; def y = 1; return x + y"); + exec("def x = null; def y = 1; return x + y"); }); } - + public void testCompoundAssignment() { // byte assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;")); @@ -406,7 +406,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(15D, exec("double x = 5.0; x += 10; return x;")); assertEquals(-5D, exec("double x = 5.0; x += -10; return x;")); } - + public void testDefCompoundAssignmentLHS() { // byte assertEquals((byte) 15, exec("def x = (byte)5; x += 10; return x;")); @@ -431,7 +431,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(15D, exec("def x = 5.0; x += 10; return x;")); assertEquals(-5D, exec("def x = 5.0; x += -10; return x;")); } - + public void testDefCompoundAssignmentRHS() { // byte assertEquals((byte) 15, exec("byte x = 5; def y = 10; x += y; return x;")); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java index 0ca72f993e5..c9954fd7171 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java @@ -318,4 +318,13 @@ public class CastTests extends ScriptTestCase { exec("def x = 5L; boolean y = (boolean) (x + x); return y"); }); } + + public void testIllegalVoidCasts() { + expectScriptThrows(IllegalArgumentException.class, () -> { + exec("def map = ['a': 1,'b': 2,'c': 3]; map.c = Collections.sort(new ArrayList(map.keySet()));"); + }); + expectScriptThrows(IllegalArgumentException.class, () -> { + exec("Map map = ['a': 1,'b': 2,'c': 3]; def x = new HashMap(); x.put(1, map.clear());"); + }); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java index 23362265474..b15a2747bd0 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java @@ -191,4 +191,13 @@ public class FactoryTests extends ScriptTestCase { assertEquals("def", script.execute()); assertEquals("def", script.execute()); } + + public void testGetterInLambda() { + FactoryTestScript.Factory factory = + scriptEngine.compile("template_test", + "IntSupplier createLambda(IntSupplier s) { return s; } createLambda(() -> params['x'] + test).getAsInt()", + FactoryTestScript.CONTEXT, Collections.emptyMap()); + FactoryTestScript script = factory.newInstance(Collections.singletonMap("x", 1)); + assertEquals(2, script.execute(1)); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java index a0df66dac73..567f4620461 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java @@ -37,10 +37,6 @@ public class ScoreTests extends ScriptTestCase { return 0; } @Override - public int freq() throws IOException { - throw new UnsupportedOperationException(); - } - @Override public DocIdSetIterator iterator() { throw new UnsupportedOperationException(); } diff --git a/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index f5d86849e56..b5caf997495 100644 --- a/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -19,6 +19,7 @@ package org.apache.lucene.queries; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; @@ -107,6 +108,11 @@ public final class BinaryDocValuesRangeQuery extends Query { }; return new ConstantScoreScorer(this, score(), iterator); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return DocValues.isCacheable(ctx, fieldName); + } }; } diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index 50f4d55750d..2b7368f9ec4 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -41,6 +41,6 @@ setup: - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - match: { hits.hits.0._id: "1" } - - is_false: hits.hits.0.inner_hits.child.hits.hits.0._index + - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"} - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" } - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index be18c62d1f2..f24a9710d29 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -164,6 +164,13 @@ final class PercolateQuery extends Query implements Accountable { }; } } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // This query uses a significant amount of memory, let's never + // cache it or compound queries that wrap it. + return false; + } }; } @@ -261,11 +268,6 @@ final class PercolateQuery extends Query implements Accountable { }; } - @Override - public final int freq() throws IOException { - return approximation.freq(); - } - @Override public final int docID() { return approximation.docID(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 35bce1f4f4f..1df6935c61a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -244,20 +244,9 @@ public class PercolatorFieldMapper extends FieldMapper { Query percolateQuery(String name, PercolateQuery.QueryStore queryStore, List documents, IndexSearcher searcher, Version indexVersion) throws IOException { IndexReader indexReader = searcher.getIndexReader(); - Tuple, Boolean> t = createCandidateQueryClauses(indexReader); - BooleanQuery.Builder candidateQuery = new BooleanQuery.Builder(); - if (t.v2() && indexVersion.onOrAfter(Version.V_6_1_0)) { - LongValuesSource valuesSource = LongValuesSource.fromIntField(minimumShouldMatchField.name()); - candidateQuery.add(new CoveringQuery(t.v1(), valuesSource), BooleanClause.Occur.SHOULD); - } else { - for (Query query : t.v1()) { - candidateQuery.add(query, BooleanClause.Occur.SHOULD); - } - } - // include extractionResultField:failed, because docs with this term have no extractedTermsField - // and otherwise we would fail to return these docs. Docs that failed query term extraction - // always need to be verified by MemoryIndex: - candidateQuery.add(new TermQuery(new Term(extractionResultField.name(), EXTRACTION_FAILED)), BooleanClause.Occur.SHOULD); + Tuple t = createCandidateQuery(indexReader, indexVersion); + Query candidateQuery = t.v1(); + boolean canUseMinimumShouldMatchField = t.v2(); Query verifiedMatchesQuery; // We can only skip the MemoryIndex verification when percolating a single non nested document. We cannot @@ -265,15 +254,55 @@ public class PercolatorFieldMapper extends FieldMapper { // ranges are extracted from IndexReader backed by a RamDirectory holding multiple documents we do // not know to which document the terms belong too and for certain queries we incorrectly emit candidate // matches as actual match. - if (t.v2() && indexReader.maxDoc() == 1) { + if (canUseMinimumShouldMatchField && indexReader.maxDoc() == 1) { verifiedMatchesQuery = new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE)); } else { verifiedMatchesQuery = new MatchNoDocsQuery("multiple or nested docs or CoveringQuery could not be used"); } - return new PercolateQuery(name, queryStore, documents, candidateQuery.build(), searcher, verifiedMatchesQuery); + return new PercolateQuery(name, queryStore, documents, candidateQuery, searcher, verifiedMatchesQuery); } - Tuple, Boolean> createCandidateQueryClauses(IndexReader indexReader) throws IOException { + Tuple createCandidateQuery(IndexReader indexReader, Version indexVersion) throws IOException { + Tuple, Map>> t = extractTermsAndRanges(indexReader); + List extractedTerms = t.v1(); + Map> encodedPointValuesByField = t.v2(); + // `1 + ` is needed to take into account the EXTRACTION_FAILED should clause + boolean canUseMinimumShouldMatchField = 1 + extractedTerms.size() + encodedPointValuesByField.size() <= + BooleanQuery.getMaxClauseCount(); + + List subQueries = new ArrayList<>(); + for (Map.Entry> entry : encodedPointValuesByField.entrySet()) { + String rangeFieldName = entry.getKey(); + List encodedPointValues = entry.getValue(); + byte[] min = encodedPointValues.get(0); + byte[] max = encodedPointValues.get(1); + Query query = BinaryRange.newIntersectsQuery(rangeField.name(), encodeRange(rangeFieldName, min, max)); + subQueries.add(query); + } + + BooleanQuery.Builder candidateQuery = new BooleanQuery.Builder(); + if (canUseMinimumShouldMatchField && indexVersion.onOrAfter(Version.V_6_1_0)) { + LongValuesSource valuesSource = LongValuesSource.fromIntField(minimumShouldMatchField.name()); + for (BytesRef extractedTerm : extractedTerms) { + subQueries.add(new TermQuery(new Term(queryTermsField.name(), extractedTerm))); + } + candidateQuery.add(new CoveringQuery(subQueries, valuesSource), BooleanClause.Occur.SHOULD); + } else { + candidateQuery.add(new TermInSetQuery(queryTermsField.name(), extractedTerms), BooleanClause.Occur.SHOULD); + for (Query subQuery : subQueries) { + candidateQuery.add(subQuery, BooleanClause.Occur.SHOULD); + } + } + // include extractionResultField:failed, because docs with this term have no extractedTermsField + // and otherwise we would fail to return these docs. Docs that failed query term extraction + // always need to be verified by MemoryIndex: + candidateQuery.add(new TermQuery(new Term(extractionResultField.name(), EXTRACTION_FAILED)), BooleanClause.Occur.SHOULD); + return new Tuple<>(candidateQuery.build(), canUseMinimumShouldMatchField); + } + + // This was extracted the method above, because otherwise it is difficult to test what terms are included in + // the query in case a CoveringQuery is used (it does not have a getter to retrieve the clauses) + Tuple, Map>> extractTermsAndRanges(IndexReader indexReader) throws IOException { List extractedTerms = new ArrayList<>(); Map> encodedPointValuesByField = new HashMap<>(); @@ -299,28 +328,7 @@ public class PercolatorFieldMapper extends FieldMapper { encodedPointValuesByField.put(info.name, encodedPointValues); } } - - final boolean canUseMinimumShouldMatchField; - final List queries = new ArrayList<>(); - if (extractedTerms.size() + encodedPointValuesByField.size() <= BooleanQuery.getMaxClauseCount()) { - canUseMinimumShouldMatchField = true; - for (BytesRef extractedTerm : extractedTerms) { - queries.add(new TermQuery(new Term(queryTermsField.name(), extractedTerm))); - } - } else { - canUseMinimumShouldMatchField = false; - queries.add(new TermInSetQuery(queryTermsField.name(), extractedTerms)); - } - - for (Map.Entry> entry : encodedPointValuesByField.entrySet()) { - String rangeFieldName = entry.getKey(); - List encodedPointValues = entry.getValue(); - byte[] min = encodedPointValues.get(0); - byte[] max = encodedPointValues.get(1); - Query query = BinaryRange.newIntersectsQuery(rangeField.name(), encodeRange(rangeFieldName, min, max)); - queries.add(query); - } - return new Tuple<>(queries, canUseMinimumShouldMatchField); + return new Tuple<>(extractedTerms, encodedPointValuesByField); } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 971be4931e6..4ecd82fd876 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.CoveringQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; @@ -505,16 +506,17 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { } try (IndexReader ir = DirectoryReader.open(directory)){ IndexSearcher percolateSearcher = new IndexSearcher(ir); - Query query = + PercolateQuery query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, v); + BooleanQuery candidateQuery = (BooleanQuery) query.getCandidateMatchesQuery(); + assertThat(candidateQuery.clauses().get(0).getQuery(), instanceOf(CoveringQuery.class)); TopDocs topDocs = shardSearcher.search(query, 10); assertEquals(2L, topDocs.totalHits); assertEquals(2, topDocs.scoreDocs.length); assertEquals(0, topDocs.scoreDocs[0].doc); assertEquals(2, topDocs.scoreDocs[1].doc); - query = new ConstantScoreQuery(query); - topDocs = shardSearcher.search(query, 10); + topDocs = shardSearcher.search(new ConstantScoreQuery(query), 10); assertEquals(2L, topDocs.totalHits); assertEquals(2, topDocs.scoreDocs.length); assertEquals(0, topDocs.scoreDocs[0].doc); @@ -526,7 +528,7 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { try (RAMDirectory directory = new RAMDirectory()) { try (IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig())) { Document document = new Document(); - for (int i = 0; i < 1025; i++) { + for (int i = 0; i < 1024; i++) { int fieldNumber = 2 + i; document.add(new StringField("field", "value" + fieldNumber, Field.Store.NO)); } @@ -693,6 +695,11 @@ public class CandidateQueryTests extends ESSingleNodeTestCase { } }; } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // doesn't matter + } }; } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index 655c0d508ec..193d0e8fe06 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.test.AbstractQueryTestCase; import org.hamcrest.Matchers; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -340,7 +341,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase, Boolean> t = fieldType.createCandidateQueryClauses(indexReader); - assertTrue(t.v2()); - List clauses = t.v1(); - clauses.sort(Comparator.comparing(Query::toString)); - assertEquals(15, clauses.size()); - assertEquals(fieldType.queryTermsField.name() + ":_field3\u0000me", clauses.get(0).toString()); - assertEquals(fieldType.queryTermsField.name() + ":_field3\u0000unhide", clauses.get(1).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000brown", clauses.get(2).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000dog", clauses.get(3).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000fox", clauses.get(4).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000jumps", clauses.get(5).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000lazy", clauses.get(6).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000over", clauses.get(7).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000quick", clauses.get(8).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field1\u0000the", clauses.get(9).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field2\u0000more", clauses.get(10).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field2\u0000some", clauses.get(11).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field2\u0000text", clauses.get(12).toString()); - assertEquals(fieldType.queryTermsField.name() + ":field4\u0000123", clauses.get(13).toString()); - assertThat(clauses.get(14).toString(), containsString(fieldName + ".range_field:, Map>> t = fieldType.extractTermsAndRanges(indexReader); + assertEquals(1, t.v2().size()); + Map> rangesMap = t.v2(); + assertEquals(1, rangesMap.size()); + + List range = rangesMap.get("number_field2"); + assertNotNull(range); + assertEquals(10, LongPoint.decodeDimension(range.get(0), 0)); + assertEquals(10, LongPoint.decodeDimension(range.get(1), 0)); + + List terms = t.v1(); + terms.sort(BytesRef::compareTo); + assertEquals(14, terms.size()); + assertEquals("_field3\u0000me", terms.get(0).utf8ToString()); + assertEquals("_field3\u0000unhide", terms.get(1).utf8ToString()); + assertEquals("field1\u0000brown", terms.get(2).utf8ToString()); + assertEquals("field1\u0000dog", terms.get(3).utf8ToString()); + assertEquals("field1\u0000fox", terms.get(4).utf8ToString()); + assertEquals("field1\u0000jumps", terms.get(5).utf8ToString()); + assertEquals("field1\u0000lazy", terms.get(6).utf8ToString()); + assertEquals("field1\u0000over", terms.get(7).utf8ToString()); + assertEquals("field1\u0000quick", terms.get(8).utf8ToString()); + assertEquals("field1\u0000the", terms.get(9).utf8ToString()); + assertEquals("field2\u0000more", terms.get(10).utf8ToString()); + assertEquals("field2\u0000some", terms.get(11).utf8ToString()); + assertEquals("field2\u0000text", terms.get(12).utf8ToString()); + assertEquals("field4\u0000123", terms.get(13).utf8ToString()); } - public void testCreateCandidateQuery_largeDocument() throws Exception { + public void testCreateCandidateQuery() throws Exception { addQueryFieldMappings(); MemoryIndex memoryIndex = new MemoryIndex(false); StringBuilder text = new StringBuilder(); - for (int i = 0; i < 1023; i++) { + for (int i = 0; i < 1022; i++) { text.append(i).append(' '); } memoryIndex.addField("field1", text.toString(), new WhitespaceAnalyzer()); memoryIndex.addField(new LongPoint("field2", 10L), new WhitespaceAnalyzer()); IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); - Tuple, Boolean> t = fieldType.createCandidateQueryClauses(indexReader); + Tuple t = fieldType.createCandidateQuery(indexReader, Version.CURRENT); assertTrue(t.v2()); - List clauses = t.v1(); - assertEquals(1024, clauses.size()); - assertThat(clauses.get(1023).toString(), containsString(fieldName + ".range_field: t = fieldType.createCandidateQuery(indexReader, Version.CURRENT); + assertTrue(t.v2()); + assertEquals(2, t.v1().clauses().size()); + assertThat(t.v1().clauses().get(0).getQuery(), instanceOf(CoveringQuery.class)); + assertThat(t.v1().clauses().get(1).getQuery(), instanceOf(TermQuery.class)); + + t = fieldType.createCandidateQuery(indexReader, Version.V_6_0_0); + assertTrue(t.v2()); + assertEquals(2, t.v1().clauses().size()); + assertThat(t.v1().clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class)); + assertThat(t.v1().clauses().get(1).getQuery(), instanceOf(TermQuery.class)); + } + + public void testExtractTermsAndRanges_numberFields() throws Exception { addQueryFieldMappings(); MemoryIndex memoryIndex = new MemoryIndex(false); @@ -385,17 +413,45 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase { IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); - Tuple, Boolean> t = fieldType.createCandidateQueryClauses(indexReader); - assertThat(t.v2(), is(true)); - List clauses = t.v1(); - assertEquals(7, clauses.size()); - assertThat(clauses.get(0).toString(), containsString(fieldName + ".range_field:, Map>> t = fieldType.extractTermsAndRanges(indexReader); + assertEquals(0, t.v1().size()); + Map> rangesMap = t.v2(); + assertEquals(7, rangesMap.size()); + + List range = rangesMap.get("number_field1"); + assertNotNull(range); + assertEquals(10, IntPoint.decodeDimension(range.get(0), 0)); + assertEquals(10, IntPoint.decodeDimension(range.get(1), 0)); + + range = rangesMap.get("number_field2"); + assertNotNull(range); + assertEquals(20L, LongPoint.decodeDimension(range.get(0), 0)); + assertEquals(20L, LongPoint.decodeDimension(range.get(1), 0)); + + range = rangesMap.get("number_field3"); + assertNotNull(range); + assertEquals(30L, LongPoint.decodeDimension(range.get(0), 0)); + assertEquals(30L, LongPoint.decodeDimension(range.get(1), 0)); + + range = rangesMap.get("number_field4"); + assertNotNull(range); + assertEquals(30F, HalfFloatPoint.decodeDimension(range.get(0), 0), 0F); + assertEquals(30F, HalfFloatPoint.decodeDimension(range.get(1), 0), 0F); + + range = rangesMap.get("number_field5"); + assertNotNull(range); + assertEquals(40F, FloatPoint.decodeDimension(range.get(0), 0), 0F); + assertEquals(40F, FloatPoint.decodeDimension(range.get(1), 0), 0F); + + range = rangesMap.get("number_field6"); + assertNotNull(range); + assertEquals(50D, DoublePoint.decodeDimension(range.get(0), 0), 0D); + assertEquals(50D, DoublePoint.decodeDimension(range.get(1), 0), 0D); + + range = rangesMap.get("number_field7"); + assertNotNull(range); + assertEquals(InetAddresses.forString("192.168.1.12"), InetAddressPoint.decode(range.get(0))); + assertEquals(InetAddresses.forString("192.168.1.24"), InetAddressPoint.decode(range.get(1))); } public void testPercolatorFieldMapper() throws Exception { diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.1.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.1.0.jar.sha1 deleted file mode 100644 index d1619f9fc19..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d9a640081289c9c50da08479ff198b579df71c26 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..e0cbc64c1a1 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +f18454bf4be4698f7e6f3a7c950f75ee3fa4436e \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.1.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.1.0.jar.sha1 deleted file mode 100644 index 89ca7249ffb..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a2ca81efc31d857fa2ade104dcdb3fed20c95ea0 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..fd6989ffad8 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +a09ec54a9434987a16a4d9b68ca301a5a3bf4123 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.1.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.1.0.jar.sha1 deleted file mode 100644 index 512ecb59fc5..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42058220ada77c4c5340e8383f62a4398e10a8ce \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..56f78f71f6c --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +f86560f04296d8d34fed535abbf8002e0985b1df \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.1.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.1.0.jar.sha1 deleted file mode 100644 index 7e68fa106c1..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2769d7f7330c78aea1edf4d8cd2eb111564c6800 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..a7abf2370cc --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +dec7d0bee4de2ac8eb5aec9cb5d244c5b405bc15 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.1.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.1.0.jar.sha1 deleted file mode 100644 index 3e59d9c24c4..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2bec616dc5bb33df9d0beddf6a9565ef14a227ff \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..67278daf3fc --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +8469b05416dcaf6ddeebcdabb551b4ccf009f7c2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.1.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.1.0.jar.sha1 deleted file mode 100644 index 55f36cb5f8e..00000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0e78e3e59b7bdf6e1aa24ff8289cc1246248f642 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1 new file mode 100644 index 00000000000..fd0478fd9f5 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1 @@ -0,0 +1 @@ +6de67186607bde2ac66b216af4e1fadffaf43e21 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java index f8b105884bf..f639c064d37 100644 --- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java +++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java @@ -41,7 +41,6 @@ import static org.hamcrest.Matchers.nullValue; transportClientRatio = 0.0, numClientNodes = 0, autoMinMasterNodes = false) -@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-azure/issues/89") public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTestCase { public AzureMinimumMasterNodesTests() { diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java index 7c51f8afe69..2990101134f 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilCommandTests.java @@ -33,7 +33,7 @@ public class EvilCommandTests extends ESTestCase { public void testCommandShutdownHook() throws Exception { final AtomicBoolean closed = new AtomicBoolean(); final boolean shouldThrow = randomBoolean(); - final Command command = new Command("test-command-shutdown-hook") { + final Command command = new Command("test-command-shutdown-hook", () -> {}) { @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index be534364d4c..f5a9469f357 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -3,12 +3,9 @@ setup: - do: indices.create: index: test - body: - settings: - index: - # initializing replicas maintain the translog causing the test to fail. - # remove once https://github.com/elastic/elasticsearch/issues/25623 is fixed. - number_of_replicas: 0 + - do: + cluster.health: + wait_for_no_initializing_shards: true --- "Translog retention": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml new file mode 100644 index 00000000000..f48aee28dc2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -0,0 +1,82 @@ +--- +"top_hits aggregation with nested documents": + - skip: + version: "5.99.99 - " + reason: "5.x nodes don't include index or id in nested top hits" + - do: + indices.create: + index: my-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + doc: + properties: + users: + type: nested + + - do: + index: + index: my-index + type: doc + id: 1 + refresh: true + body: | + { + "group" : "fans", + "users" : [ + { + "first" : "John", + "last" : "Smith" + }, + { + "first" : "Alice", + "last" : "White" + } + ] + } + + - do: + index: + index: my-index + type: doc + id: 2 + refresh: true + body: | + { + "group" : "fans", + "users" : [ + { + "first" : "Mark", + "last" : "Doe" + } + ] + } + + - do: + search: + body: + aggs: + to-users: + nested: + path: users + aggs: + users: + top_hits: + sort: "users.last.keyword" + + - match: { hits.total: 2 } + - length: { aggregations.to-users.users.hits.hits: 3 } + - match: { aggregations.to-users.users.hits.hits.0._id: "2" } + - match: { aggregations.to-users.users.hits.hits.0._index: my-index } + - match: { aggregations.to-users.users.hits.hits.0._nested.field: users } + - match: { aggregations.to-users.users.hits.hits.0._nested.offset: 0 } + - match: { aggregations.to-users.users.hits.hits.1._id: "1" } + - match: { aggregations.to-users.users.hits.hits.1._index: my-index } + - match: { aggregations.to-users.users.hits.hits.1._nested.field: users } + - match: { aggregations.to-users.users.hits.hits.1._nested.offset: 0 } + - match: { aggregations.to-users.users.hits.hits.2._id: "1" } + - match: { aggregations.to-users.users.hits.hits.2._index: my-index } + - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } + - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index 4264083c13f..80d3f924d2b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -34,9 +34,9 @@ setup: - match: { hits.hits.0._index: "test" } - match: { hits.hits.0._type: "type_1" } - match: { hits.hits.0._id: "1" } - - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._index - - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._type - - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._id + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._index: "test" } + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "type1" } + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.field: "nested_field" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 } - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.child diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index bf2ffc5236e..708a95e4a49 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -347,15 +347,15 @@ public class ElasticsearchAssertions { public static void assertAllSuccessful(BroadcastResponse response) { assertNoFailures(response); - assertThat("Expected all shards successful but got successful [" + response.getSuccessfulShards() + "] total [" + response.getTotalShards() + "]", - response.getTotalShards(), equalTo(response.getSuccessfulShards())); + assertThat("Expected all shards successful", + response.getSuccessfulShards(), equalTo(response.getTotalShards())); assertVersionSerializable(response); } public static void assertAllSuccessful(SearchResponse response) { assertNoFailures(response); - assertThat("Expected all shards successful but got successful [" + response.getSuccessfulShards() + "] total [" + response.getTotalShards() + "]", - response.getTotalShards(), equalTo(response.getSuccessfulShards())); + assertThat("Expected all shards successful", + response.getSuccessfulShards(), equalTo(response.getTotalShards())); assertVersionSerializable(response); }