diff --git a/.gitignore b/.gitignore index af7e64fc895..d1810a5a83f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,11 @@ nbactions.xml .gradle/ build/ +# gradle wrapper +/gradle/ +gradlew +gradlew.bat + # maven stuff (to be removed when trunk becomes 4.x) *-execution-hints.log target/ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy index 0c2e37ab821..0fefecc1446 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy @@ -18,6 +18,7 @@ */ package org.elasticsearch.gradle.doc +import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.test.RestTestPlugin import org.gradle.api.Project import org.gradle.api.Task @@ -30,9 +31,19 @@ public class DocsTestPlugin extends RestTestPlugin { @Override public void apply(Project project) { super.apply(project) + Map defaultSubstitutions = [ + /* These match up with the asciidoc syntax for substitutions but + * the values may differ. In particular {version} needs to resolve + * to the version being built for testing but needs to resolve to + * the last released version for docs. */ + '\\{version\\}': + VersionProperties.elasticsearch.replace('-SNAPSHOT', ''), + '\\{lucene_version\\}' : VersionProperties.lucene, + ] Task listSnippets = project.tasks.create('listSnippets', SnippetsTask) listSnippets.group 'Docs' listSnippets.description 'List each snippet' + listSnippets.defaultSubstitutions = defaultSubstitutions listSnippets.perSnippet { println(it.toString()) } Task listConsoleCandidates = project.tasks.create( @@ -40,6 +51,7 @@ public class DocsTestPlugin extends RestTestPlugin { listConsoleCandidates.group 'Docs' listConsoleCandidates.description 'List snippets that probably should be marked // CONSOLE' + listConsoleCandidates.defaultSubstitutions = defaultSubstitutions listConsoleCandidates.perSnippet { if ( it.console != null // Already marked, nothing to do @@ -47,19 +59,17 @@ public class DocsTestPlugin extends RestTestPlugin { ) { return } - List languages = [ - // This language should almost always be marked console - 'js', - // These are often curl commands that should be converted but - // are probably false positives - 'sh', 'shell', - ] - if (false == languages.contains(it.language)) { - return + if ( // js almost always should be `// CONSOLE` + it.language == 'js' || + // snippets containing `curl` *probably* should + // be `// CONSOLE` + it.curl) { + println(it.toString()) } - println(it.toString()) } - project.tasks.create('buildRestTests', RestTestsFromSnippetsTask) + Task buildRestTests = project.tasks.create( + 'buildRestTests', RestTestsFromSnippetsTask) + buildRestTests.defaultSubstitutions = defaultSubstitutions } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy index fc7604ad1fd..dc4e6f5f70a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy @@ -146,6 +146,9 @@ public class RestTestsFromSnippetsTask extends SnippetsTask { void emitDo(String method, String pathAndQuery, String body, String catchPart, List warnings, boolean inSetup) { def (String path, String query) = pathAndQuery.tokenize('?') + if (path == null) { + path = '' // Catch requests to the root... + } current.println(" - do:") if (catchPart != null) { current.println(" catch: $catchPart") diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy index 8c3524a9b9f..41f74b45be1 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy @@ -22,6 +22,7 @@ package org.elasticsearch.gradle.doc import org.gradle.api.DefaultTask import org.gradle.api.InvalidUserDataException import org.gradle.api.file.ConfigurableFileTree +import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles import org.gradle.api.tasks.TaskAction @@ -60,6 +61,12 @@ public class SnippetsTask extends DefaultTask { exclude 'build' } + /** + * Substitutions done on every snippet's contents. + */ + @Input + Map defaultSubstitutions = [:] + @TaskAction public void executeTask() { /* @@ -75,21 +82,39 @@ public class SnippetsTask extends DefaultTask { Closure emit = { snippet.contents = contents.toString() contents = null + Closure doSubstitution = { String pattern, String subst -> + /* + * $body is really common but it looks like a + * backreference so we just escape it here to make the + * tests cleaner. + */ + subst = subst.replace('$body', '\\$body') + // \n is a new line.... + subst = subst.replace('\\n', '\n') + snippet.contents = snippet.contents.replaceAll( + pattern, subst) + } + defaultSubstitutions.each doSubstitution if (substitutions != null) { - substitutions.each { String pattern, String subst -> - /* - * $body is really common but it looks like a - * backreference so we just escape it here to make the - * tests cleaner. - */ - subst = subst.replace('$body', '\\$body') - // \n is a new line.... - subst = subst.replace('\\n', '\n') - snippet.contents = snippet.contents.replaceAll( - pattern, subst) - } + substitutions.each doSubstitution substitutions = null } + if (snippet.language == null) { + throw new InvalidUserDataException("$snippet: " + + "Snippet missing a language. This is required by " + + "Elasticsearch's doc testing infrastructure so we " + + "be sure we don't accidentally forget to test a " + + "snippet.") + } + // Try to detect snippets that contain `curl` + if (snippet.language == 'sh' || snippet.language == 'shell') { + snippet.curl = snippet.contents.contains('curl') + if (snippet.console == false && snippet.curl == false) { + throw new InvalidUserDataException("$snippet: " + + "No need for NOTCONSOLE if snippet doesn't " + + "contain `curl`.") + } + } perSnippet(snippet) snippet = null } @@ -107,7 +132,7 @@ public class SnippetsTask extends DefaultTask { } return } - matcher = line =~ /\[source,(\w+)]\s*/ + matcher = line =~ /\["?source"?,\s*"?(\w+)"?(,.*)?].*/ if (matcher.matches()) { lastLanguage = matcher.group(1) lastLanguageLine = lineNumber @@ -250,6 +275,7 @@ public class SnippetsTask extends DefaultTask { String language = null String catchPart = null String setup = null + boolean curl List warnings = new ArrayList() @Override @@ -285,6 +311,9 @@ public class SnippetsTask extends DefaultTask { if (testSetup) { result += '// TESTSETUP' } + if (curl) { + result += '(curl)' + } return result } } diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index d50f362a73e..14c2bc8ca5a 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -470,7 +470,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 883d21154bd..9587b4e6b2c 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -38,10 +38,12 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; @@ -53,14 +55,17 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportChannelResponseHandler; import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; @@ -69,6 +74,7 @@ import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; @@ -115,9 +121,12 @@ public abstract class TransportReplicationAction< this.transportPrimaryAction = actionName + "[p]"; this.transportReplicaAction = actionName + "[r]"; transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, new OperationTransportHandler()); - transportService.registerRequestHandler(transportPrimaryAction, request, executor, new PrimaryOperationTransportHandler()); + transportService.registerRequestHandler(transportPrimaryAction, () -> new ConcreteShardRequest<>(request), executor, + new PrimaryOperationTransportHandler()); // we must never reject on because of thread pool capacity on replicas - transportService.registerRequestHandler(transportReplicaAction, replicaRequest, executor, true, true, + transportService.registerRequestHandler(transportReplicaAction, + () -> new ConcreteShardRequest<>(replicaRequest), + executor, true, true, new ReplicaOperationTransportHandler()); this.transportOptions = transportOptions(); @@ -163,7 +172,7 @@ public abstract class TransportReplicationAction< /** * Synchronous replica operation on nodes with replica copies. This is done under the lock form - * {@link #acquireReplicaOperationLock(ShardId, long, ActionListener)}. + * {@link #acquireReplicaOperationLock(ShardId, long, String, ActionListener)}. */ protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest); @@ -230,33 +239,36 @@ public abstract class TransportReplicationAction< } } - class PrimaryOperationTransportHandler implements TransportRequestHandler { + class PrimaryOperationTransportHandler implements TransportRequestHandler> { @Override - public void messageReceived(final Request request, final TransportChannel channel) throws Exception { + public void messageReceived(final ConcreteShardRequest request, final TransportChannel channel) throws Exception { throw new UnsupportedOperationException("the task parameter is required for this operation"); } @Override - public void messageReceived(Request request, TransportChannel channel, Task task) { - new AsyncPrimaryAction(request, channel, (ReplicationTask) task).run(); + public void messageReceived(ConcreteShardRequest request, TransportChannel channel, Task task) { + new AsyncPrimaryAction(request.request, request.targetAllocationID, channel, (ReplicationTask) task).run(); } } class AsyncPrimaryAction extends AbstractRunnable implements ActionListener { private final Request request; + /** targetAllocationID of the shard this request is meant for */ + private final String targetAllocationID; private final TransportChannel channel; private final ReplicationTask replicationTask; - AsyncPrimaryAction(Request request, TransportChannel channel, ReplicationTask replicationTask) { + AsyncPrimaryAction(Request request, String targetAllocationID, TransportChannel channel, ReplicationTask replicationTask) { this.request = request; + this.targetAllocationID = targetAllocationID; this.channel = channel; this.replicationTask = replicationTask; } @Override protected void doRun() throws Exception { - acquirePrimaryShardReference(request.shardId(), this); + acquirePrimaryShardReference(request.shardId(), targetAllocationID, this); } @Override @@ -271,7 +283,9 @@ public abstract class TransportReplicationAction< final ShardRouting primary = primaryShardReference.routingEntry(); assert primary.relocating() : "indexShard is marked as relocated but routing isn't" + primary; DiscoveryNode relocatingNode = clusterService.state().nodes().get(primary.relocatingNodeId()); - transportService.sendRequest(relocatingNode, transportPrimaryAction, request, transportOptions, + transportService.sendRequest(relocatingNode, transportPrimaryAction, + new ConcreteShardRequest<>(request, primary.allocationId().getRelocationId()), + transportOptions, new TransportChannelResponseHandler(logger, channel, "rerouting indexing to target primary " + primary, TransportReplicationAction.this::newResponseInstance) { @@ -391,15 +405,17 @@ public abstract class TransportReplicationAction< } } - class ReplicaOperationTransportHandler implements TransportRequestHandler { + class ReplicaOperationTransportHandler implements TransportRequestHandler> { @Override - public void messageReceived(final ReplicaRequest request, final TransportChannel channel) throws Exception { + public void messageReceived(final ConcreteShardRequest request, final TransportChannel channel) + throws Exception { throw new UnsupportedOperationException("the task parameter is required for this operation"); } @Override - public void messageReceived(ReplicaRequest request, TransportChannel channel, Task task) throws Exception { - new AsyncReplicaAction(request, channel, (ReplicationTask) task).run(); + public void messageReceived(ConcreteShardRequest requestWithAID, TransportChannel channel, Task task) + throws Exception { + new AsyncReplicaAction(requestWithAID.request, requestWithAID.targetAllocationID, channel, (ReplicationTask) task).run(); } } @@ -417,6 +433,8 @@ public abstract class TransportReplicationAction< private final class AsyncReplicaAction extends AbstractRunnable implements ActionListener { private final ReplicaRequest request; + // allocation id of the replica this request is meant for + private final String targetAllocationID; private final TransportChannel channel; /** * The task on the node with the replica shard. @@ -426,10 +444,11 @@ public abstract class TransportReplicationAction< // something we want to avoid at all costs private final ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext()); - AsyncReplicaAction(ReplicaRequest request, TransportChannel channel, ReplicationTask task) { + AsyncReplicaAction(ReplicaRequest request, String targetAllocationID, TransportChannel channel, ReplicationTask task) { this.request = request; this.channel = channel; this.task = task; + this.targetAllocationID = targetAllocationID; } @Override @@ -464,7 +483,9 @@ public abstract class TransportReplicationAction< String extraMessage = "action [" + transportReplicaAction + "], request[" + request + "]"; TransportChannelResponseHandler handler = new TransportChannelResponseHandler<>(logger, channel, extraMessage, () -> TransportResponse.Empty.INSTANCE); - transportService.sendRequest(clusterService.localNode(), transportReplicaAction, request, handler); + transportService.sendRequest(clusterService.localNode(), transportReplicaAction, + new ConcreteShardRequest<>(request, targetAllocationID), + handler); } @Override @@ -501,7 +522,7 @@ public abstract class TransportReplicationAction< protected void doRun() throws Exception { setPhase(task, "replica"); assert request.shardId() != null : "request shardId must be set"; - acquireReplicaOperationLock(request.shardId(), request.primaryTerm(), this); + acquireReplicaOperationLock(request.shardId(), request.primaryTerm(), targetAllocationID, this); } /** @@ -598,7 +619,7 @@ public abstract class TransportReplicationAction< logger.trace("send action [{}] on primary [{}] for request [{}] with cluster state version [{}] to [{}] ", transportPrimaryAction, request.shardId(), request, state.version(), primary.currentNodeId()); } - performAction(node, transportPrimaryAction, true); + performAction(node, transportPrimaryAction, true, new ConcreteShardRequest<>(request, primary.allocationId().getId())); } private void performRemoteAction(ClusterState state, ShardRouting primary, DiscoveryNode node) { @@ -620,7 +641,7 @@ public abstract class TransportReplicationAction< request.shardId(), request, state.version(), primary.currentNodeId()); } setPhase(task, "rerouted"); - performAction(node, actionName, false); + performAction(node, actionName, false, request); } private boolean retryIfUnavailable(ClusterState state, ShardRouting primary) { @@ -671,8 +692,9 @@ public abstract class TransportReplicationAction< } } - private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction) { - transportService.sendRequest(node, action, request, transportOptions, new TransportResponseHandler() { + private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction, + final TransportRequest requestToPerform) { + transportService.sendRequest(node, action, requestToPerform, transportOptions, new TransportResponseHandler() { @Override public Response newInstance() { @@ -700,7 +722,7 @@ public abstract class TransportReplicationAction< (org.apache.logging.log4j.util.Supplier) () -> new ParameterizedMessage( "received an error from node [{}] for request [{}], scheduling a retry", node.getId(), - request), + requestToPerform), exp); retry(exp); } else { @@ -794,7 +816,8 @@ public abstract class TransportReplicationAction< * tries to acquire reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally * and replication of the operation to all replica shards is completed / failed (see {@link ReplicationOperation}). */ - protected void acquirePrimaryShardReference(ShardId shardId, ActionListener onReferenceAcquired) { + protected void acquirePrimaryShardReference(ShardId shardId, String allocationId, + ActionListener onReferenceAcquired) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); // we may end up here if the cluster state used to route the primary is so stale that the underlying @@ -804,6 +827,10 @@ public abstract class TransportReplicationAction< throw new ReplicationOperation.RetryOnPrimaryException(indexShard.shardId(), "actual shard is not a primary " + indexShard.routingEntry()); } + final String actualAllocationId = indexShard.routingEntry().allocationId().getId(); + if (actualAllocationId.equals(allocationId) == false) { + throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId); + } ActionListener onAcquired = new ActionListener() { @Override @@ -823,9 +850,14 @@ public abstract class TransportReplicationAction< /** * tries to acquire an operation on replicas. The lock is closed as soon as replication is completed on the node. */ - protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, ActionListener onLockAcquired) { + protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, final String allocationId, + ActionListener onLockAcquired) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); + final String actualAllocationId = indexShard.routingEntry().allocationId().getId(); + if (actualAllocationId.equals(allocationId) == false) { + throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId); + } indexShard.acquireReplicaOperationLock(primaryTerm, onLockAcquired, executor); } @@ -888,7 +920,8 @@ public abstract class TransportReplicationAction< listener.onFailure(new NoNodeAvailableException("unknown node [" + nodeId + "]")); return; } - transportService.sendRequest(node, transportReplicaAction, request, transportOptions, + transportService.sendRequest(node, transportReplicaAction, + new ConcreteShardRequest<>(request, replica.allocationId().getId()), transportOptions, new ActionListenerResponseHandler<>(listener, () -> TransportResponse.Empty.INSTANCE)); } @@ -930,6 +963,72 @@ public abstract class TransportReplicationAction< } } + /** a wrapper class to encapsulate a request when being sent to a specific allocation id **/ + public static final class ConcreteShardRequest extends TransportRequest { + + /** {@link AllocationId#getId()} of the shard this request is sent to **/ + private String targetAllocationID; + + private R request; + + ConcreteShardRequest(Supplier requestSupplier) { + request = requestSupplier.get(); + // null now, but will be populated by reading from the streams + targetAllocationID = null; + } + + ConcreteShardRequest(R request, String targetAllocationID) { + Objects.requireNonNull(request); + Objects.requireNonNull(targetAllocationID); + this.request = request; + this.targetAllocationID = targetAllocationID; + } + + @Override + public void setParentTask(String parentTaskNode, long parentTaskId) { + request.setParentTask(parentTaskNode, parentTaskId); + } + + @Override + public void setParentTask(TaskId taskId) { + request.setParentTask(taskId); + } + + @Override + public TaskId getParentTask() { + return request.getParentTask(); + } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return request.createTask(id, type, action, parentTaskId); + } + + @Override + public String getDescription() { + return "[" + request.getDescription() + "] for aID [" + targetAllocationID + "]"; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + targetAllocationID = in.readString(); + request.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(targetAllocationID); + request.writeTo(out); + } + + public R getRequest() { + return request; + } + + public String getTargetAllocationID() { + return targetAllocationID; + } + } + /** * Sets the current phase on the task if it isn't null. Pulled into its own * method because its more convenient that way. diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java b/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java index c4cd5d908af..4a0aaec68c2 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JarHell.java @@ -168,7 +168,7 @@ public class JarHell { if (path.toString().endsWith(".jar")) { if (!seenJars.add(path)) { logger.debug("excluding duplicate classpath element: {}", path); - continue; // we can't fail because of sheistiness with joda-time + continue; } logger.debug("examining jar: {}", path); try (JarFile file = new JarFile(path.toString())) { @@ -271,12 +271,6 @@ public class JarHell { "class: " + clazz + System.lineSeparator() + "exists multiple times in jar: " + jarpath + " !!!!!!!!!"); } else { - if (clazz.startsWith("org.apache.log4j")) { - return; // go figure, jar hell for what should be System.out.println... - } - if (clazz.equals("org.joda.time.base.BaseDateTime")) { - return; // apparently this is intentional... clean this up - } if (clazz.startsWith("org.apache.logging.log4j.core.impl.ThrowableProxy")) { /* * deliberate to hack around a bug in Log4j diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 2ca165308b1..2d960ce0450 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -27,8 +27,8 @@ import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Nullable; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -88,6 +88,11 @@ public class RoutingTable implements Iterable, Diffable usages) { + private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation, + ImmutableOpenMap usages, boolean subtractLeavingShards) { DiskUsage usage = usages.get(node.nodeId()); if (usage == null) { // If there is no usage, and we have other nodes in the cluster, @@ -293,7 +297,7 @@ public class DiskThresholdDecider extends AllocationDecider { } if (diskThresholdSettings.includeRelocations()) { - long relocatingShardsSize = sizeOfRelocatingShards(node, allocation, true, usage.getPath()); + long relocatingShardsSize = sizeOfRelocatingShards(node, allocation, subtractLeavingShards, usage.getPath()); DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().getName(), usage.getPath(), usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize); if (logger.isTraceEnabled()) { diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index 12f4805ba2f..c22e12b3ce8 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -551,10 +551,6 @@ public class Setting extends ToXContentToBytes { return new Setting<>(key, defaultValueFn, Booleans::parseBooleanExact, properties); } - public static Setting byteSizeSetting(String key, String percentage, Property... properties) { - return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties); - } - public static Setting byteSizeSetting(String key, ByteSizeValue value, Property... properties) { return byteSizeSetting(key, (s) -> value.toString(), properties); } @@ -591,6 +587,49 @@ public class Setting extends ToXContentToBytes { return value; } + /** + * Creates a setting which specifies a memory size. This can either be + * specified as an absolute bytes value or as a percentage of the heap + * memory. + * + * @param key the key for the setting + * @param defaultValue the default value for this setting + * @param properties properties properties for this setting like scope, filtering... + * @return the setting object + */ + public static Setting memorySizeSetting(String key, ByteSizeValue defaultValue, Property... properties) { + return memorySizeSetting(key, (s) -> defaultValue.toString(), properties); + } + + + /** + * Creates a setting which specifies a memory size. This can either be + * specified as an absolute bytes value or as a percentage of the heap + * memory. + * + * @param key the key for the setting + * @param defaultValue a function that supplies the default value for this setting + * @param properties properties properties for this setting like scope, filtering... + * @return the setting object + */ + public static Setting memorySizeSetting(String key, Function defaultValue, Property... properties) { + return new Setting<>(key, defaultValue, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties); + } + + /** + * Creates a setting which specifies a memory size. This can either be + * specified as an absolute bytes value or as a percentage of the heap + * memory. + * + * @param key the key for the setting + * @param defaultPercentage the default value of this setting as a percentage of the heap memory + * @param properties properties properties for this setting like scope, filtering... + * @return the setting object + */ + public static Setting memorySizeSetting(String key, String defaultPercentage, Property... properties) { + return new Setting<>(key, (s) -> defaultPercentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties); + } + public static Setting positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) { return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties); } diff --git a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java index eed357dee74..6a68dda0272 100644 --- a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java +++ b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java @@ -46,7 +46,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable { public static final Setting TYPE_SETTING = new Setting<>("cache.recycler.page.type", Type.CONCURRENT.name(), Type::parse, Property.NodeScope); public static final Setting LIMIT_HEAP_SETTING = - Setting.byteSizeSetting("cache.recycler.page.limit.heap", "10%", Property.NodeScope); + Setting.memorySizeSetting("cache.recycler.page.limit.heap", "10%", Property.NodeScope); public static final Setting WEIGHT_BYTES_SETTING = Setting.doubleSetting("cache.recycler.page.weight.bytes", 1d, 0d, Property.NodeScope); public static final Setting WEIGHT_LONG_SETTING = diff --git a/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java new file mode 100644 index 00000000000..19213dffe2a --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/analysis/MinHashTokenFilterFactory.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.minhash.MinHashFilterFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; + +import java.util.HashMap; +import java.util.Map; + +/** + * TokenFilterFactoryAdapter for {@link MinHashFilterFactory} + * + */ +public class MinHashTokenFilterFactory extends AbstractTokenFilterFactory { + + private final MinHashFilterFactory minHashFilterFactory; + + public MinHashTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + minHashFilterFactory = new MinHashFilterFactory(convertSettings(settings)); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return minHashFilterFactory.create(tokenStream); + } + + private Map convertSettings(Settings settings) { + Map settingMap = new HashMap<>(); + settingMap.put("hashCount", settings.get("hash_count")); + settingMap.put("bucketCount", settings.get("bucket_count")); + settingMap.put("hashSetSize", settings.get("hash_set_size")); + settingMap.put("withRotation", settings.get("with_rotation")); + return settingMap; + } +} diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index 95fe0094bad..9ed374db212 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -25,8 +25,11 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptSettings; import java.io.IOException; import java.util.Objects; @@ -42,11 +45,18 @@ public class QueryParseContext implements ParseFieldMatcherSupplier { private final XContentParser parser; private final IndicesQueriesRegistry indicesQueriesRegistry; private final ParseFieldMatcher parseFieldMatcher; + private final String defaultScriptLanguage; public QueryParseContext(IndicesQueriesRegistry registry, XContentParser parser, ParseFieldMatcher parseFieldMatcher) { + this(Script.DEFAULT_SCRIPT_LANG, registry, parser, parseFieldMatcher); + } + + public QueryParseContext(String defaultScriptLanguage, IndicesQueriesRegistry registry, XContentParser parser, + ParseFieldMatcher parseFieldMatcher) { this.indicesQueriesRegistry = Objects.requireNonNull(registry, "indices queries registry cannot be null"); this.parser = Objects.requireNonNull(parser, "parser cannot be null"); this.parseFieldMatcher = Objects.requireNonNull(parseFieldMatcher, "parse field matcher cannot be null"); + this.defaultScriptLanguage = defaultScriptLanguage; } public XContentParser parser() { @@ -127,4 +137,12 @@ public class QueryParseContext implements ParseFieldMatcherSupplier { public ParseFieldMatcher getParseFieldMatcher() { return parseFieldMatcher; } + + /** + * Returns the default scripting language, that should be used if scripts don't specify the script language + * explicitly. + */ + public String getDefaultScriptLanguage() { + return defaultScriptLanguage; + } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index d8ec9ef2a47..f12605088e6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptSettings; /** * Context object used to rewrite {@link QueryBuilder} instances into simplified version. @@ -101,9 +102,18 @@ public class QueryRewriteContext implements ParseFieldMatcherSupplier { /** * Returns a new {@link QueryParseContext} that wraps the provided parser, using the ParseFieldMatcher settings that - * are configured in the index settings + * are configured in the index settings. The default script language will always default to Painless. */ public QueryParseContext newParseContext(XContentParser parser) { return new QueryParseContext(indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher()); } + + /** + * Returns a new {@link QueryParseContext} like {@link #newParseContext(XContentParser)} with the only diffence, that + * the default script language will default to what has been set in the 'script.legacy.default_lang' setting. + */ + public QueryParseContext newParseContextWithLegacyScriptLanguage(XContentParser parser) { + String defaultScriptLanguage = ScriptSettings.getLegacyDefaultLang(indexSettings.getNodeSettings()); + return new QueryParseContext(defaultScriptLanguage, indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher()); + } } diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index e6e902e68f8..3ff924b28db 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -106,7 +106,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder // skip } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, parseContext.getParseFieldMatcher()); + script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); } @@ -116,7 +116,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, parseContext.getParseFieldMatcher()); + script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java index 4cbf71f2943..e2fbc5955d7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java @@ -115,7 +115,7 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder INDEX_BUFFER_SIZE_SETTING = Setting.byteSizeSetting("indices.memory.index_buffer_size", "10%", Property.NodeScope); + public static final Setting INDEX_BUFFER_SIZE_SETTING = + Setting.memorySizeSetting("indices.memory.index_buffer_size", "10%", Property.NodeScope); /** Only applies when indices.memory.index_buffer_size is a %, to set a floor on the actual size in bytes (default: 48 MB). */ public static final Setting MIN_INDEX_BUFFER_SIZE_SETTING = Setting.byteSizeSetting("indices.memory.min_index_buffer_size", diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 70b9443e043..08dcd48a56f 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -49,13 +49,13 @@ import java.util.function.Predicate; public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable { - public static final Setting INDICES_CACHE_QUERY_SIZE_SETTING = Setting.byteSizeSetting( - "indices.queries.cache.size", "10%", Property.NodeScope); - public static final Setting INDICES_CACHE_QUERY_COUNT_SETTING = Setting.intSetting( - "indices.queries.cache.count", 10000, 1, Property.NodeScope); + public static final Setting INDICES_CACHE_QUERY_SIZE_SETTING = + Setting.memorySizeSetting("indices.queries.cache.size", "10%", Property.NodeScope); + public static final Setting INDICES_CACHE_QUERY_COUNT_SETTING = + Setting.intSetting("indices.queries.cache.count", 10000, 1, Property.NodeScope); // enables caching on all segments instead of only the larger ones, for testing only - public static final Setting INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = Setting.boolSetting( - "indices.queries.cache.all_segments", false, Property.NodeScope); + public static final Setting INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = + Setting.boolSetting("indices.queries.cache.all_segments", false, Property.NodeScope); private final LRUQueryCache cache; private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap(); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index 5229de3f16c..f78ccb22c9d 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -72,7 +72,7 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo public static final Setting INDEX_CACHE_REQUEST_ENABLED_SETTING = Setting.boolSetting("index.requests.cache.enable", true, Property.Dynamic, Property.IndexScope); public static final Setting INDICES_CACHE_QUERY_SIZE = - Setting.byteSizeSetting("indices.requests.cache.size", "1%", Property.NodeScope); + Setting.memorySizeSetting("indices.requests.cache.size", "1%", Property.NodeScope); public static final Setting INDICES_CACHE_QUERY_EXPIRE = Setting.positiveTimeSetting("indices.requests.cache.expire", new TimeValue(0), Property.NodeScope); diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index 52647c8b8ff..5dd0203d617 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -90,6 +90,7 @@ import org.elasticsearch.index.analysis.LithuanianAnalyzerProvider; import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory; import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory; import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.MinHashTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenizerFactory; import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider; @@ -214,6 +215,7 @@ public final class AnalysisModule { tokenFilters.register("edgeNGram", EdgeNGramTokenFilterFactory::new); tokenFilters.register("edge_ngram", EdgeNGramTokenFilterFactory::new); tokenFilters.register("shingle", ShingleTokenFilterFactory::new); + tokenFilters.register("min_hash", MinHashTokenFilterFactory::new); tokenFilters.register("unique", UniqueTokenFilterFactory::new); tokenFilters.register("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new)); tokenFilters.register("trim", TrimTokenFilterFactory::new); diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index 65571482093..715cf47a6ef 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -47,24 +47,24 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { private final ConcurrentMap breakers = new ConcurrentHashMap<>(); public static final Setting TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING = - Setting.byteSizeSetting("indices.breaker.total.limit", "70%", Property.Dynamic, Property.NodeScope); + Setting.memorySizeSetting("indices.breaker.total.limit", "70%", Property.Dynamic, Property.NodeScope); public static final Setting FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING = - Setting.byteSizeSetting("indices.breaker.fielddata.limit", "60%", Property.Dynamic, Property.NodeScope); + Setting.memorySizeSetting("indices.breaker.fielddata.limit", "60%", Property.Dynamic, Property.NodeScope); public static final Setting FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("indices.breaker.fielddata.overhead", 1.03d, 0.0d, Property.Dynamic, Property.NodeScope); public static final Setting FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING = new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING = - Setting.byteSizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope); + Setting.memorySizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_TYPE_SETTING = new Setting<>("indices.breaker.request.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope); public static final Setting IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING = - Setting.byteSizeSetting("network.breaker.inflight_requests.limit", "100%", Property.Dynamic, Property.NodeScope); + Setting.memorySizeSetting("network.breaker.inflight_requests.limit", "100%", Property.Dynamic, Property.NodeScope); public static final Setting IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("network.breaker.inflight_requests.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope); public static final Setting IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_TYPE_SETTING = diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java index 81e9f3fac5f..c5a8be9b114 100644 --- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -51,7 +51,7 @@ import java.util.function.ToLongBiFunction; public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener, Releasable{ public static final Setting INDICES_FIELDDATA_CACHE_SIZE_KEY = - Setting.byteSizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), Property.NodeScope); + Setting.memorySizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), Property.NodeScope); private final IndexFieldDataCache.Listener indicesFieldDataCacheListener; private final Cache cache; diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java index 0b4f8d281d0..54cd34d6742 100644 --- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java @@ -43,7 +43,7 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; /** * A command for the plugin cli to remove a plugin from elasticsearch. */ -class RemovePluginCommand extends SettingCommand { +final class RemovePluginCommand extends SettingCommand { private final OptionSpec arguments; @@ -64,14 +64,16 @@ class RemovePluginCommand extends SettingCommand { terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "..."); - Path pluginDir = env.pluginsFile().resolve(pluginName); + final Path pluginDir = env.pluginsFile().resolve(pluginName); if (Files.exists(pluginDir) == false) { - throw new UserException(ExitCodes.USAGE, "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins"); + throw new UserException( + ExitCodes.USAGE, + "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins"); } - List pluginPaths = new ArrayList<>(); + final List pluginPaths = new ArrayList<>(); - Path pluginBinDir = env.binFile().resolve(pluginName); + final Path pluginBinDir = env.binFile().resolve(pluginName); if (Files.exists(pluginBinDir)) { if (Files.isDirectory(pluginBinDir) == false) { throw new UserException(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory"); @@ -81,10 +83,19 @@ class RemovePluginCommand extends SettingCommand { } terminal.println(VERBOSE, "Removing: " + pluginDir); - Path tmpPluginDir = env.pluginsFile().resolve(".removing-" + pluginName); + final Path tmpPluginDir = env.pluginsFile().resolve(".removing-" + pluginName); Files.move(pluginDir, tmpPluginDir, StandardCopyOption.ATOMIC_MOVE); pluginPaths.add(tmpPluginDir); IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()])); + + // we preserve the config files in case the user is upgrading the plugin, but we print + // a message so the user knows in case they want to remove manually + final Path pluginConfigDir = env.configFile().resolve(pluginName); + if (Files.exists(pluginConfigDir)) { + terminal.println( + "-> Preserving plugin config files [" + pluginConfigDir + "] in case of upgrade, delete manually if not needed"); + } } + } diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index 5888bd07c07..c028913d343 100644 --- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -54,9 +54,9 @@ public class FsRepository extends BlobStoreRepository { public static final Setting REPOSITORIES_LOCATION_SETTING = new Setting<>("repositories.fs.location", LOCATION_SETTING, Function.identity(), Property.NodeScope); public static final Setting CHUNK_SIZE_SETTING = - Setting.byteSizeSetting("chunk_size", "-1", Property.NodeScope); + Setting.byteSizeSetting("chunk_size", new ByteSizeValue(-1), Property.NodeScope); public static final Setting REPOSITORIES_CHUNK_SIZE_SETTING = - Setting.byteSizeSetting("repositories.fs.chunk_size", "-1", Property.NodeScope); + Setting.byteSizeSetting("repositories.fs.chunk_size", new ByteSizeValue(-1), Property.NodeScope); public static final Setting COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope); public static final Setting REPOSITORIES_COMPRESS_SETTING = Setting.boolSetting("repositories.fs.compress", false, Property.NodeScope); diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java index cedae963ca4..94abb43bc06 100644 --- a/core/src/main/java/org/elasticsearch/script/Script.java +++ b/core/src/main/java/org/elasticsearch/script/Script.java @@ -44,6 +44,7 @@ import java.util.Objects; public final class Script implements ToXContent, Writeable { public static final ScriptType DEFAULT_TYPE = ScriptType.INLINE; + public static final String DEFAULT_SCRIPT_LANG = "painless"; private String script; private ScriptType type; @@ -60,7 +61,7 @@ public final class Script implements ToXContent, Writeable { this(script, ScriptType.INLINE, null, null); } - public Script(String script, ScriptType type, @Nullable String lang, @Nullable Map params) { + public Script(String script, ScriptType type, String lang, @Nullable Map params) { this(script, type, lang, params, null); } @@ -78,14 +79,14 @@ public final class Script implements ToXContent, Writeable { * when serializing the script back to xcontent. */ @SuppressWarnings("unchecked") - public Script(String script, ScriptType type, @Nullable String lang, @Nullable Map params, + public Script(String script, ScriptType type, String lang, @Nullable Map params, @Nullable XContentType contentType) { if (contentType != null && type != ScriptType.INLINE) { throw new IllegalArgumentException("The parameter contentType only makes sense for inline scripts"); } this.script = Objects.requireNonNull(script); this.type = Objects.requireNonNull(type); - this.lang = lang; + this.lang = lang == null ? DEFAULT_SCRIPT_LANG : lang; this.params = (Map) params; this.contentType = contentType; } @@ -135,7 +136,7 @@ public final class Script implements ToXContent, Writeable { * @return The type of script -- inline, stored, or file. */ public ScriptType getType() { - return type == null ? DEFAULT_TYPE : type; + return type; } /** @@ -196,7 +197,7 @@ public final class Script implements ToXContent, Writeable { token = parser.nextToken(); } if (token == XContentParser.Token.VALUE_STRING) { - return new Script(parser.text()); + return new Script(parser.text(), ScriptType.INLINE, lang, null); } if (token != XContentParser.Token.START_OBJECT) { throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 84e0a900d25..9e61f39378e 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -92,8 +92,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust public static final Setting SCRIPT_MAX_COMPILATIONS_PER_MINUTE = Setting.intSetting("script.max_compilations_per_minute", 15, 0, Property.Dynamic, Property.NodeScope); - private final String defaultLang; - private final Collection scriptEngines; private final Map scriptEnginesByLang; private final Map scriptEnginesByExt; @@ -131,8 +129,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust this.scriptContextRegistry = scriptContextRegistry; int cacheMaxSize = SCRIPT_CACHE_SIZE_SETTING.get(settings); - this.defaultLang = scriptSettings.getDefaultScriptLanguageSetting().get(settings); - CacheBuilder cacheBuilder = CacheBuilder.builder(); if (cacheMaxSize >= 0) { cacheBuilder.setMaximumWeight(cacheMaxSize); @@ -222,11 +218,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust } String lang = script.getLang(); - - if (lang == null) { - lang = defaultLang; - } - ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang); if (canExecuteScript(lang, script.getType(), scriptContext) == false) { throw new IllegalStateException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled"); @@ -285,7 +276,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust throw new IllegalArgumentException("The parameter script (Script) must not be null."); } - String lang = script.getLang() == null ? defaultLang : script.getLang(); + String lang = script.getLang(); ScriptType type = script.getType(); //script.getScript() could return either a name or code for a script, //but we check for a file script name first and an indexed script name second @@ -364,9 +355,8 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust } private String validateScriptLanguage(String scriptLang) { - if (scriptLang == null) { - scriptLang = defaultLang; - } else if (scriptEnginesByLang.containsKey(scriptLang) == false) { + Objects.requireNonNull(scriptLang); + if (scriptEnginesByLang.containsKey(scriptLang) == false) { throw new IllegalArgumentException("script_lang not supported [" + scriptLang + "]"); } return scriptLang; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java index 9c98b8d1e8c..1cb2b356245 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java @@ -32,7 +32,16 @@ import java.util.function.Function; public class ScriptSettings { - public static final String DEFAULT_LANG = "painless"; + static final String LEGACY_DEFAULT_LANG = "groovy"; + + /** + * The default script language to use for scripts that are stored in documents that have no script lang set explicitly. + * This setting is legacy setting and only applies for indices created on ES versions prior to version 5.0 + * + * This constant will be removed in the next major release. + */ + @Deprecated + public static final String LEGACY_SCRIPT_SETTING = "script.legacy.default_lang"; private static final Map> SCRIPT_TYPE_SETTING_MAP; @@ -49,7 +58,7 @@ public class ScriptSettings { private final Map> scriptContextSettingMap; private final List> scriptLanguageSettings; - private final Setting defaultScriptLanguageSetting; + private final Setting defaultLegacyScriptLanguageSetting; public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) { Map> scriptContextSettingMap = contextSettings(scriptContextRegistry); @@ -58,8 +67,8 @@ public class ScriptSettings { List> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry); this.scriptLanguageSettings = Collections.unmodifiableList(scriptLanguageSettings); - this.defaultScriptLanguageSetting = new Setting<>("script.default_lang", DEFAULT_LANG, setting -> { - if (!DEFAULT_LANG.equals(setting) && !scriptEngineRegistry.getRegisteredLanguages().containsKey(setting)) { + this.defaultLegacyScriptLanguageSetting = new Setting<>(LEGACY_SCRIPT_SETTING, LEGACY_DEFAULT_LANG, setting -> { + if (!LEGACY_DEFAULT_LANG.equals(setting) && !scriptEngineRegistry.getRegisteredLanguages().containsKey(setting)) { throw new IllegalArgumentException("unregistered default language [" + setting + "]"); } return setting; @@ -160,7 +169,7 @@ public class ScriptSettings { settings.addAll(SCRIPT_TYPE_SETTING_MAP.values()); settings.addAll(scriptContextSettingMap.values()); settings.addAll(scriptLanguageSettings); - settings.add(defaultScriptLanguageSetting); + settings.add(defaultLegacyScriptLanguageSetting); return settings; } @@ -168,7 +177,11 @@ public class ScriptSettings { return scriptLanguageSettings; } - public Setting getDefaultScriptLanguageSetting() { - return defaultScriptLanguageSetting; + public Setting getDefaultLegacyScriptLanguageSetting() { + return defaultLegacyScriptLanguageSetting; + } + + public static String getLegacyDefaultLang(Settings settings) { + return settings.get(LEGACY_SCRIPT_SETTING, ScriptSettings.LEGACY_DEFAULT_LANG); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java index 1ae31e09ba0..e669ee8b9d9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java @@ -19,11 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -65,16 +65,17 @@ public class GeoHashGridParser extends GeoPointValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token == XContentParser.Token.VALUE_NUMBER || token == XContentParser.Token.VALUE_STRING) { - if (parseFieldMatcher.match(currentFieldName, GeoHashGridParams.FIELD_PRECISION)) { + if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_PRECISION)) { otherOptions.put(GeoHashGridParams.FIELD_PRECISION, parser.intValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, GeoHashGridParams.FIELD_SIZE)) { + } else if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_SIZE)) { otherOptions.put(GeoHashGridParams.FIELD_SIZE, parser.intValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, GeoHashGridParams.FIELD_SHARD_SIZE)) { + } else if (context.matchField(currentFieldName, GeoHashGridParams.FIELD_SHARD_SIZE)) { otherOptions.put(GeoHashGridParams.FIELD_SHARD_SIZE, parser.intValue()); return true; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java index e3a3ea75762..952a0e2568f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java @@ -19,11 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -79,10 +79,11 @@ public class DateHistogramParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token.isValue()) { - if (parseFieldMatcher.match(currentFieldName, Histogram.INTERVAL_FIELD)) { + if (context.matchField(currentFieldName, Histogram.INTERVAL_FIELD)) { if (token == XContentParser.Token.VALUE_STRING) { otherOptions.put(Histogram.INTERVAL_FIELD, new DateHistogramInterval(parser.text())); return true; @@ -90,13 +91,13 @@ public class DateHistogramParser extends NumericValuesSourceParser { otherOptions.put(Histogram.INTERVAL_FIELD, parser.longValue()); return true; } - } else if (parseFieldMatcher.match(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) { otherOptions.put(Histogram.MIN_DOC_COUNT_FIELD, parser.longValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.KEYED_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.KEYED_FIELD)) { otherOptions.put(Histogram.KEYED_FIELD, parser.booleanValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.OFFSET_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.OFFSET_FIELD)) { if (token == XContentParser.Token.VALUE_STRING) { otherOptions.put(Histogram.OFFSET_FIELD, DateHistogramAggregationBuilder.parseStringOffset(parser.text())); @@ -109,7 +110,7 @@ public class DateHistogramParser extends NumericValuesSourceParser { return false; } } else if (token == XContentParser.Token.START_OBJECT) { - if (parseFieldMatcher.match(currentFieldName, Histogram.ORDER_FIELD)) { + if (context.matchField(currentFieldName, Histogram.ORDER_FIELD)) { InternalOrder order = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -127,9 +128,10 @@ public class DateHistogramParser extends NumericValuesSourceParser { } otherOptions.put(Histogram.ORDER_FIELD, order); return true; - } else if (parseFieldMatcher.match(currentFieldName, ExtendedBounds.EXTENDED_BOUNDS_FIELD)) { + } else if (context.matchField(currentFieldName, ExtendedBounds.EXTENDED_BOUNDS_FIELD)) { try { - otherOptions.put(ExtendedBounds.EXTENDED_BOUNDS_FIELD, ExtendedBounds.PARSER.apply(parser, () -> parseFieldMatcher)); + otherOptions.put(ExtendedBounds.EXTENDED_BOUNDS_FIELD, + ExtendedBounds.PARSER.apply(parser, context::getParseFieldMatcher)); } catch (Exception e) { throw new ParsingException(parser.getTokenLocation(), "Error parsing [{}]", e, aggregationName); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java index 69aed3e733a..f27677a1a66 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java @@ -19,13 +19,13 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -85,26 +85,27 @@ public class HistogramParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token.isValue()) { - if (parseFieldMatcher.match(currentFieldName, Histogram.INTERVAL_FIELD)) { + if (context.matchField(currentFieldName, Histogram.INTERVAL_FIELD)) { otherOptions.put(Histogram.INTERVAL_FIELD, parser.doubleValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.MIN_DOC_COUNT_FIELD)) { otherOptions.put(Histogram.MIN_DOC_COUNT_FIELD, parser.longValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.KEYED_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.KEYED_FIELD)) { otherOptions.put(Histogram.KEYED_FIELD, parser.booleanValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.OFFSET_FIELD)) { + } else if (context.matchField(currentFieldName, Histogram.OFFSET_FIELD)) { otherOptions.put(Histogram.OFFSET_FIELD, parser.doubleValue()); return true; } else { return false; } } else if (token == XContentParser.Token.START_OBJECT) { - if (parseFieldMatcher.match(currentFieldName, Histogram.ORDER_FIELD)) { + if (context.matchField(currentFieldName, Histogram.ORDER_FIELD)) { InternalOrder order = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -122,8 +123,8 @@ public class HistogramParser extends NumericValuesSourceParser { } otherOptions.put(Histogram.ORDER_FIELD, order); return true; - } else if (parseFieldMatcher.match(currentFieldName, Histogram.EXTENDED_BOUNDS_FIELD)) { - double[] bounds = EXTENDED_BOUNDS_PARSER.apply(parser, () -> parseFieldMatcher); + } else if (context.matchField(currentFieldName, Histogram.EXTENDED_BOUNDS_FIELD)) { + double[] bounds = EXTENDED_BOUNDS_PARSER.apply(parser, context::getParseFieldMatcher); otherOptions.put(Histogram.EXTENDED_BOUNDS_FIELD, bounds); return true; } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java index fff81db1301..5d6844ebbd2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.bucket.missing; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -35,8 +35,8 @@ public class MissingParser extends AnyValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java index 6ebd413d2d4..c8cb2c76715 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -65,20 +66,21 @@ public class RangeParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token == XContentParser.Token.START_ARRAY) { - if (parseFieldMatcher.match(currentFieldName, RangeAggregator.RANGES_FIELD)) { + if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) { List ranges = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - Range range = parseRange(parser, parseFieldMatcher); + Range range = parseRange(parser, context.getParseFieldMatcher()); ranges.add(range); } otherOptions.put(RangeAggregator.RANGES_FIELD, ranges); return true; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (parseFieldMatcher.match(currentFieldName, RangeAggregator.KEYED_FIELD)) { + if (context.matchField(currentFieldName, RangeAggregator.KEYED_FIELD)) { boolean keyed = parser.booleanValue(); otherOptions.put(RangeAggregator.KEYED_FIELD, keyed); return true; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java index 16cb909ea0f..677731d64ef 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.range.geodistance; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; @@ -30,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser; import org.elasticsearch.search.aggregations.support.GeoPointParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -110,28 +110,29 @@ public class GeoDistanceParser extends GeoPointValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { - if (geoPointParser.token(aggregationName, currentFieldName, token, parser, parseFieldMatcher, otherOptions)) { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); + if (geoPointParser.token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { return true; } else if (token == XContentParser.Token.VALUE_STRING) { - if (parseFieldMatcher.match(currentFieldName, UNIT_FIELD)) { + if (context.matchField(currentFieldName, UNIT_FIELD)) { DistanceUnit unit = DistanceUnit.fromString(parser.text()); otherOptions.put(UNIT_FIELD, unit); return true; - } else if (parseFieldMatcher.match(currentFieldName, DISTANCE_TYPE_FIELD)) { + } else if (context.matchField(currentFieldName, DISTANCE_TYPE_FIELD)) { GeoDistance distanceType = GeoDistance.fromString(parser.text()); otherOptions.put(DISTANCE_TYPE_FIELD, distanceType); return true; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (parseFieldMatcher.match(currentFieldName, RangeAggregator.KEYED_FIELD)) { + if (context.matchField(currentFieldName, RangeAggregator.KEYED_FIELD)) { boolean keyed = parser.booleanValue(); otherOptions.put(RangeAggregator.KEYED_FIELD, keyed); return true; } } else if (token == XContentParser.Token.START_ARRAY) { - if (parseFieldMatcher.match(currentFieldName, RangeAggregator.RANGES_FIELD)) { + if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) { List ranges = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String fromAsStr = null; @@ -144,17 +145,17 @@ public class GeoDistanceParser extends GeoPointValuesSourceParser { if (token == XContentParser.Token.FIELD_NAME) { toOrFromOrKey = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (parseFieldMatcher.match(toOrFromOrKey, Range.FROM_FIELD)) { + if (context.matchField(toOrFromOrKey, Range.FROM_FIELD)) { from = parser.doubleValue(); - } else if (parseFieldMatcher.match(toOrFromOrKey, Range.TO_FIELD)) { + } else if (context.matchField(toOrFromOrKey, Range.TO_FIELD)) { to = parser.doubleValue(); } } else if (token == XContentParser.Token.VALUE_STRING) { - if (parseFieldMatcher.match(toOrFromOrKey, Range.KEY_FIELD)) { + if (context.matchField(toOrFromOrKey, Range.KEY_FIELD)) { key = parser.text(); - } else if (parseFieldMatcher.match(toOrFromOrKey, Range.FROM_FIELD)) { + } else if (context.matchField(toOrFromOrKey, Range.FROM_FIELD)) { fromAsStr = parser.text(); - } else if (parseFieldMatcher.match(toOrFromOrKey, Range.TO_FIELD)) { + } else if (context.matchField(toOrFromOrKey, Range.TO_FIELD)) { toAsStr = parser.text(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java index 8445fb2d459..5d95f0dd494 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.BytesValuesSourceParser; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator; import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder.Range; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -102,21 +103,22 @@ public class IpRangeParser extends BytesValuesSourceParser { @Override protected boolean token(String aggregationName, String currentFieldName, - Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, - Map otherOptions) throws IOException { - if (parseFieldMatcher.match(currentFieldName, RangeAggregator.RANGES_FIELD)) { + Token token, + XContentParseContext context, + Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); + if (context.matchField(currentFieldName, RangeAggregator.RANGES_FIELD)) { if (parser.currentToken() != Token.START_ARRAY) { throw new ParsingException(parser.getTokenLocation(), "[ranges] must be passed as an array, but got a " + token); } List ranges = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - Range range = parseRange(parser, parseFieldMatcher); + Range range = parseRange(parser, context.getParseFieldMatcher()); ranges.add(range); } otherOptions.put(RangeAggregator.RANGES_FIELD, ranges); return true; - } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.KEYED_FIELD)) { + } else if (context.matchField(parser.currentName(), RangeAggregator.KEYED_FIELD)) { otherOptions.put(RangeAggregator.KEYED_FIELD, parser.booleanValue()); return true; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java index f495071f6d2..a62035d7234 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.aggregations.bucket.sampler; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -57,20 +57,21 @@ public class DiversifiedSamplerParser extends AnyValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token == XContentParser.Token.VALUE_NUMBER) { - if (parseFieldMatcher.match(currentFieldName, SamplerAggregator.SHARD_SIZE_FIELD)) { + if (context.matchField(currentFieldName, SamplerAggregator.SHARD_SIZE_FIELD)) { int shardSize = parser.intValue(); otherOptions.put(SamplerAggregator.SHARD_SIZE_FIELD, shardSize); return true; - } else if (parseFieldMatcher.match(currentFieldName, SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD)) { + } else if (context.matchField(currentFieldName, SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD)) { int maxDocsPerValue = parser.intValue(); otherOptions.put(SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD, maxDocsPerValue); return true; } } else if (token == XContentParser.Token.VALUE_STRING) { - if (parseFieldMatcher.match(currentFieldName, SamplerAggregator.EXECUTION_HINT_FIELD)) { + if (context.matchField(currentFieldName, SamplerAggregator.EXECUTION_HINT_FIELD)) { String executionHint = parser.text(); otherOptions.put(SamplerAggregator.EXECUTION_HINT_FIELD, executionHint); return true; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java index ba87f0917a0..0f08cf0a0a3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.significant; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -33,6 +32,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.AbstractTermsParser; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -81,17 +81,18 @@ public class SignificantTermsParser extends AbstractTermsParser { } @Override - public boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, Token token, - String currentFieldName, Map otherOptions) throws IOException { + public boolean parseSpecial(String aggregationName, XContentParseContext context, Token token, + String currentFieldName, Map otherOptions) throws IOException { if (token == XContentParser.Token.START_OBJECT) { SignificanceHeuristicParser significanceHeuristicParser = significanceHeuristicParserRegistry - .lookupReturningNullIfNotFound(currentFieldName, parseFieldMatcher); + .lookupReturningNullIfNotFound(currentFieldName, context.getParseFieldMatcher()); if (significanceHeuristicParser != null) { - SignificanceHeuristic significanceHeuristic = significanceHeuristicParser.parse(parser, parseFieldMatcher); + SignificanceHeuristic significanceHeuristic = significanceHeuristicParser.parse(context); otherOptions.put(SignificantTermsAggregationBuilder.HEURISTIC, significanceHeuristic); return true; - } else if (parseFieldMatcher.match(currentFieldName, SignificantTermsAggregationBuilder.BACKGROUND_FILTER)) { - QueryParseContext queryParseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher); + } else if (context.matchField(currentFieldName, SignificantTermsAggregationBuilder.BACKGROUND_FILTER)) { + QueryParseContext queryParseContext = new QueryParseContext(context.getDefaultScriptLanguage(), queriesRegistry, + context.getParser(), context.getParseFieldMatcher()); Optional filter = queryParseContext.parseInnerQueryBuilder(); if (filter.isPresent()) { otherOptions.put(SignificantTermsAggregationBuilder.BACKGROUND_FILTER, filter.get()); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java index d8610dc05c8..3ae26639aa9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java @@ -22,12 +22,12 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import java.io.IOException; @@ -113,13 +113,13 @@ public class GND extends NXYSignificanceHeuristic { } @Override - public SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) - throws IOException, QueryShardException { + public SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { + XContentParser parser = context.getParser(); String givenName = parser.currentName(); boolean backgroundIsSuperset = true; XContentParser.Token token = parser.nextToken(); while (!token.equals(XContentParser.Token.END_OBJECT)) { - if (parseFieldMatcher.match(parser.currentName(), BACKGROUND_IS_SUPERSET)) { + if (context.matchField(parser.currentName(), BACKGROUND_IS_SUPERSET)) { parser.nextToken(); backgroundIsSuperset = parser.booleanValue(); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java index d426b146620..58f8060a108 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/JLHScore.java @@ -22,12 +22,12 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import java.io.IOException; @@ -104,8 +104,9 @@ public class JLHScore extends SignificanceHeuristic { return builder; } - public static SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + public static SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { + XContentParser parser = context.getParser(); // move to the closing bracket if (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { throw new ElasticsearchParseException( diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java index 3036c57865d..d6064ca37fd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java @@ -23,12 +23,12 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import java.io.IOException; @@ -152,17 +152,18 @@ public abstract class NXYSignificanceHeuristic extends SignificanceHeuristic { public abstract static class NXYParser implements SignificanceHeuristicParser { @Override - public SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + public SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { + XContentParser parser = context.getParser(); String givenName = parser.currentName(); boolean includeNegatives = false; boolean backgroundIsSuperset = true; XContentParser.Token token = parser.nextToken(); while (!token.equals(XContentParser.Token.END_OBJECT)) { - if (parseFieldMatcher.match(parser.currentName(), INCLUDE_NEGATIVES_FIELD)) { + if (context.matchField(parser.currentName(), INCLUDE_NEGATIVES_FIELD)) { parser.nextToken(); includeNegatives = parser.booleanValue(); - } else if (parseFieldMatcher.match(parser.currentName(), BACKGROUND_IS_SUPERSET)) { + } else if (context.matchField(parser.currentName(), BACKGROUND_IS_SUPERSET)) { parser.nextToken(); backgroundIsSuperset = parser.booleanValue(); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java index 7bc117a0ec8..c7e5c7ead6f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/PercentageScore.java @@ -22,12 +22,12 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import java.io.IOException; @@ -56,8 +56,9 @@ public class PercentageScore extends SignificanceHeuristic { return builder; } - public static SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + public static SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { + XContentParser parser = context.getParser(); // move to the closing bracket if (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { throw new ElasticsearchParseException("failed to parse [percentage] significance heuristic. expected an empty object, but got [{}] instead", parser.currentToken()); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java index 1f99ebad216..c933f9ef596 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLoggerFactory; @@ -35,6 +34,7 @@ import org.elasticsearch.script.Script.ScriptField; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -146,8 +146,9 @@ public class ScriptHeuristic extends SignificanceHeuristic { return Objects.equals(script, other.script); } - public static SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + public static SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { + XContentParser parser = context.getParser(); String heuristicName = parser.currentName(); Script script = null; XContentParser.Token token; @@ -156,8 +157,8 @@ public class ScriptHeuristic extends SignificanceHeuristic { if (token.equals(XContentParser.Token.FIELD_NAME)) { currentFieldName = parser.currentName(); } else { - if (parseFieldMatcher.match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, parseFieldMatcher); + if (context.matchField(currentFieldName, ScriptField.SCRIPT)) { + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else { throw new ElasticsearchParseException("failed to parse [{}] significance heuristic. unknown object [{}]", heuristicName, currentFieldName); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java index 1e1f4bfd486..26fd552a6b1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicParser.java @@ -20,9 +20,9 @@ package org.elasticsearch.search.aggregations.bucket.significant.heuristics; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import java.io.IOException; @@ -31,6 +31,5 @@ import java.io.IOException; */ @FunctionalInterface public interface SignificanceHeuristicParser { - SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException, - ParsingException; + SignificanceHeuristic parse(XContentParseContext context) throws IOException, ParsingException; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java index 3f27c4f1c6f..a106cea3a15 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java @@ -20,13 +20,13 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -89,47 +89,48 @@ public abstract class AbstractTermsParser extends AnyValuesSourceParser { Map otherOptions); @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { - if (incExcParser.token(currentFieldName, token, parser, parseFieldMatcher, otherOptions)) { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); + if (incExcParser.token(currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { return true; } else if (token == XContentParser.Token.VALUE_STRING) { - if (parseFieldMatcher.match(currentFieldName, EXECUTION_HINT_FIELD_NAME)) { + if (context.matchField(currentFieldName, EXECUTION_HINT_FIELD_NAME)) { otherOptions.put(EXECUTION_HINT_FIELD_NAME, parser.text()); return true; - } else if (parseFieldMatcher.match(currentFieldName, SubAggCollectionMode.KEY)) { - otherOptions.put(SubAggCollectionMode.KEY, SubAggCollectionMode.parse(parser.text(), parseFieldMatcher)); + } else if (context.matchField(currentFieldName, SubAggCollectionMode.KEY)) { + otherOptions.put(SubAggCollectionMode.KEY, SubAggCollectionMode.parse(parser.text(), context.getParseFieldMatcher())); return true; - } else if (parseFieldMatcher.match(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) { + } else if (context.matchField(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) { otherOptions.put(REQUIRED_SIZE_FIELD_NAME, parser.intValue()); return true; - } else if (parseSpecial(aggregationName, parser, parseFieldMatcher, token, currentFieldName, otherOptions)) { + } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) { return true; } } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (parseFieldMatcher.match(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) { + if (context.matchField(currentFieldName, REQUIRED_SIZE_FIELD_NAME)) { otherOptions.put(REQUIRED_SIZE_FIELD_NAME, parser.intValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, SHARD_SIZE_FIELD_NAME)) { + } else if (context.matchField(currentFieldName, SHARD_SIZE_FIELD_NAME)) { otherOptions.put(SHARD_SIZE_FIELD_NAME, parser.intValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, MIN_DOC_COUNT_FIELD_NAME)) { + } else if (context.matchField(currentFieldName, MIN_DOC_COUNT_FIELD_NAME)) { otherOptions.put(MIN_DOC_COUNT_FIELD_NAME, parser.longValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, SHARD_MIN_DOC_COUNT_FIELD_NAME)) { + } else if (context.matchField(currentFieldName, SHARD_MIN_DOC_COUNT_FIELD_NAME)) { otherOptions.put(SHARD_MIN_DOC_COUNT_FIELD_NAME, parser.longValue()); return true; - } else if (parseSpecial(aggregationName, parser, parseFieldMatcher, token, currentFieldName, otherOptions)) { + } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) { return true; } - } else if (parseSpecial(aggregationName, parser, parseFieldMatcher, token, currentFieldName, otherOptions)) { + } else if (parseSpecial(aggregationName, context, token, currentFieldName, otherOptions)) { return true; } return false; } - public abstract boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, - XContentParser.Token token, String currentFieldName, Map otherOptions) throws IOException; + public abstract boolean parseSpecial(String aggregationName, XContentParseContext context, + Token token, String currentFieldName, Map otherOptions) throws IOException; protected abstract TermsAggregator.BucketCountThresholds getDefaultBucketCountThresholds(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java index 2a67dbe2218..bf8b06ab65a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -27,6 +26,7 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -75,15 +75,16 @@ public class TermsParser extends AbstractTermsParser { } @Override - public boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, Token token, - String currentFieldName, Map otherOptions) throws IOException { + public boolean parseSpecial(String aggregationName, XContentParseContext context, Token token, + String currentFieldName, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token == XContentParser.Token.START_OBJECT) { - if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) { + if (context.matchField(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) { otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser))); return true; } } else if (token == XContentParser.Token.START_ARRAY) { - if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) { + if (context.matchField(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) { List orderElements = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { @@ -98,7 +99,7 @@ public class TermsParser extends AbstractTermsParser { return true; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR)) { + if (context.matchField(currentFieldName, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR)) { otherOptions.put(TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR, parser.booleanValue()); return true; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java index b4f9261b1eb..bc6f762295c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.avg; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -38,8 +38,8 @@ public class AvgParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java index 3a2e6a2072a..e40e0767994 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java @@ -20,10 +20,9 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -51,13 +50,13 @@ public class CardinalityParser extends AnyValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { if (token.isValue()) { - if (parseFieldMatcher.match(currentFieldName, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD)) { - otherOptions.put(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD, parser.longValue()); + if (context.matchField(currentFieldName, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD)) { + otherOptions.put(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD, context.getParser().longValue()); return true; - } else if (parseFieldMatcher.match(currentFieldName, REHASH)) { + } else if (context.matchField(currentFieldName, REHASH)) { // ignore return true; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java index 7420fc0149e..c42de23949b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java @@ -20,10 +20,10 @@ package org.elasticsearch.search.aggregations.metrics.geobounds; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -48,11 +48,11 @@ public class GeoBoundsParser extends GeoPointValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (parseFieldMatcher.match(currentFieldName, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD)) { - otherOptions.put(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD, parser.booleanValue()); + if (context.matchField(currentFieldName, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD)) { + otherOptions.put(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD, context.getParser().booleanValue()); return true; } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java index 6c9e9ba67b0..8e88a11c6b6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java @@ -20,10 +20,9 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.GeoPointValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -40,8 +39,8 @@ public class GeoCentroidParser extends GeoPointValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java index d2ddd4daa08..f0290e93fa9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.max; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -38,8 +38,8 @@ public class MaxParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java index 194c08fc49b..4381ca41899 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java @@ -19,10 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.min; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -39,8 +38,8 @@ public class MinParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java index ec145754a04..053a415c971 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java @@ -21,10 +21,10 @@ package org.elasticsearch.search.aggregations.metrics.percentiles; import com.carrotsearch.hppc.DoubleArrayList; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -45,10 +45,11 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse } @Override - protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, Token token, + XContentParseContext context, Map otherOptions) throws IOException { + XContentParser parser = context.getParser(); if (token == XContentParser.Token.START_ARRAY) { - if (parseFieldMatcher.match(currentFieldName, keysField())) { + if (context.matchField(currentFieldName, keysField())) { DoubleArrayList values = new DoubleArrayList(10); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { double value = parser.doubleValue(); @@ -61,7 +62,7 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse return false; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (parseFieldMatcher.match(currentFieldName, KEYED_FIELD)) { + if (context.matchField(currentFieldName, KEYED_FIELD)) { boolean keyed = parser.booleanValue(); otherOptions.put(KEYED_FIELD, keyed); return true; @@ -80,7 +81,7 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (parseFieldMatcher.match(currentFieldName, COMPRESSION_FIELD)) { + if (context.matchField(currentFieldName, COMPRESSION_FIELD)) { double compression = parser.doubleValue(); otherOptions.put(COMPRESSION_FIELD, compression); } else { @@ -96,7 +97,7 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (parseFieldMatcher.match(currentFieldName, NUMBER_SIGNIFICANT_DIGITS_FIELD)) { + if (context.matchField(currentFieldName, NUMBER_SIGNIFICANT_DIGITS_FIELD)) { int numberOfSignificantValueDigits = parser.intValue(); otherOptions.put(NUMBER_SIGNIFICANT_DIGITS_FIELD, numberOfSignificantValueDigits); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java index ec0b2aef613..244881a5155 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java @@ -232,13 +232,13 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.VALUE_STRING) { if (context.getParseFieldMatcher().match(currentFieldName, INIT_SCRIPT_FIELD)) { - initScript = Script.parse(parser, context.getParseFieldMatcher()); + initScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, MAP_SCRIPT_FIELD)) { - mapScript = Script.parse(parser, context.getParseFieldMatcher()); + mapScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, COMBINE_SCRIPT_FIELD)) { - combineScript = Script.parse(parser, context.getParseFieldMatcher()); + combineScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, REDUCE_SCRIPT_FIELD)) { - reduceScript = Script.parse(parser, context.getParseFieldMatcher()); + reduceScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (token == XContentParser.Token.START_OBJECT && context.getParseFieldMatcher().match(currentFieldName, PARAMS_FIELD)) { params = parser.map(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java index eacfc0068b4..60e3d2ef0aa 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.stats; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -38,8 +38,8 @@ public class StatsParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java index c650847360f..9644d26e93a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.stats.extended; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -38,11 +38,11 @@ public class ExtendedStatsParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { - if (parseFieldMatcher.match(currentFieldName, ExtendedStatsAggregator.SIGMA_FIELD)) { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { + if (context.matchField(currentFieldName, ExtendedStatsAggregator.SIGMA_FIELD)) { if (token.isValue()) { - otherOptions.put(ExtendedStatsAggregator.SIGMA_FIELD, parser.doubleValue()); + otherOptions.put(ExtendedStatsAggregator.SIGMA_FIELD, context.getParser().doubleValue()); return true; } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java index 6edc6cc8905..ee82829b0a7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java @@ -19,9 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.sum; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -38,8 +38,8 @@ public class SumParser extends NumericValuesSourceParser { } @Override - protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java index 0c21f78aa06..828d5679846 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java @@ -622,7 +622,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder otherOptions) throws IOException { + protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java index cee17076e5d..cd7b1bb828e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java @@ -179,7 +179,7 @@ public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggr } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) { gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation()); } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "]."); @@ -201,7 +201,7 @@ public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggr } } else if (token == XContentParser.Token.START_OBJECT) { if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) { Map map = parser.map(); bucketsPathsMap = new HashMap<>(); @@ -260,4 +260,4 @@ public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggr public String getWriteableName() { return NAME; } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java index 97cf02d69a1..e3b42376728 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java @@ -142,7 +142,7 @@ public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAg } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) { gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation()); } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "]."); @@ -164,7 +164,7 @@ public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAg } } else if (token == XContentParser.Token.START_OBJECT) { if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) { Map map = parser.map(); bucketsPathsMap = new HashMap<>(); @@ -219,4 +219,4 @@ public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAg public String getWriteableName() { return NAME; } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java index 51d2ea2e8c9..57eea9ccf65 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.support; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; @@ -95,6 +94,8 @@ public abstract class AbstractValuesSourceParser Object missing = null; DateTimeZone timezone = null; Map otherOptions = new HashMap<>(); + XContentParseContext parserContext = + new XContentParseContext(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); XContentParser.Token token; String currentFieldName = null; @@ -126,22 +127,22 @@ public abstract class AbstractValuesSourceParser + valueType + "]. It can only work on value of type [" + targetValueType + "]"); } - } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { + } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) { throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]."); } - } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { + } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) { throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]."); } } else if (scriptable && token == XContentParser.Token.START_OBJECT) { if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.getParseFieldMatcher()); - } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); + } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) { throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]."); } - } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) { + } else if (!token(aggregationName, currentFieldName, token, parserContext, otherOptions)) { throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]."); } @@ -184,8 +185,7 @@ public abstract class AbstractValuesSourceParser * the target type of the final value output by the aggregation * @param otherOptions * a {@link Map} containing the extra options parsed by the - * {@link #token(String, String, org.elasticsearch.common.xcontent.XContentParser.Token, - * XContentParser, ParseFieldMatcher, Map)} + * {@link #token(String, String, XContentParser.Token, XContentParseContext, Map)} * method * @return the created factory */ @@ -203,10 +203,8 @@ public abstract class AbstractValuesSourceParser * the name of the current field being parsed * @param token * the current token for the parser - * @param parser - * the parser - * @param parseFieldMatcher - * the {@link ParseFieldMatcher} to use to match field names + * @param context + * the query context * @param otherOptions * a {@link Map} of options to be populated by successive calls * to this method which will then be passed to the @@ -217,6 +215,6 @@ public abstract class AbstractValuesSourceParser * @throws IOException * if an error occurs whilst parsing */ - protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser, - ParseFieldMatcher parseFieldMatcher, Map otherOptions) throws IOException; + protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, + XContentParseContext context, Map otherOptions) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java new file mode 100644 index 00000000000..07c33f1f473 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/XContentParseContext.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.support; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; + +/** + * A minimal context for parsing xcontent into aggregation builders. + * Only a minimal set of dependencies and settings are available. + */ +public final class XContentParseContext { + + private final XContentParser parser; + + private final ParseFieldMatcher parseFieldMatcher; + + private final String defaultScriptLanguage; + + public XContentParseContext(XContentParser parser, ParseFieldMatcher parseFieldMatcher, String defaultScriptLanguage) { + this.parser = parser; + this.parseFieldMatcher = parseFieldMatcher; + this.defaultScriptLanguage = defaultScriptLanguage; + } + + public XContentParser getParser() { + return parser; + } + + public ParseFieldMatcher getParseFieldMatcher() { + return parseFieldMatcher; + } + + public String getDefaultScriptLanguage() { + return defaultScriptLanguage; + } + + /** + * Returns whether the parse field we're looking for matches with the found field name. + * + * Helper that delegates to {@link ParseFieldMatcher#match(String, ParseField)}. + */ + public boolean matchField(String fieldName, ParseField parseField) { + return parseFieldMatcher.match(fieldName, parseField); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 7cfc998836d..309e49448e9 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1273,7 +1273,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ currentFieldName = parser.currentName(); } else if (token.isValue()) { if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else if (context.getParseFieldMatcher().match(currentFieldName, IGNORE_FAILURE_FIELD)) { ignoreFailure = parser.booleanValue(); } else { @@ -1282,7 +1282,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } } else if (token == XContentParser.Token.START_OBJECT) { if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) { - script = Script.parse(parser, context.getParseFieldMatcher()); + script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index 8c0436361c2..0a7cb5e1b36 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -244,7 +244,7 @@ public class ScriptSortBuilder extends SortBuilder { currentName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { if (parseField.match(currentName, ScriptField.SCRIPT)) { - script = Script.parse(parser, parseField); + script = Script.parse(parser, parseField, context.getDefaultScriptLanguage()); } else if (parseField.match(currentName, NESTED_FILTER_FIELD)) { nestedFilter = context.parseInnerQueryBuilder(); } else { @@ -260,7 +260,7 @@ public class ScriptSortBuilder extends SortBuilder { } else if (parseField.match(currentName, NESTED_PATH_FIELD)) { nestedPath = parser.text(); } else if (parseField.match(currentName, ScriptField.SCRIPT)) { - script = Script.parse(parser, parseField); + script = Script.parse(parser, parseField, context.getDefaultScriptLanguage()); } else { throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]"); } diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java index 934fdae254b..1a11e3f4803 100644 --- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java @@ -69,6 +69,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.replication.TransportReplicationActionTests; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -117,7 +118,6 @@ import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.instanceOf; @ClusterScope(scope = Scope.SUITE, numClientNodes = 1, minNumDataNodes = 2) public class IndicesRequestIT extends ESIntegTestCase { @@ -638,8 +638,7 @@ public class IndicesRequestIT extends ESIntegTestCase { assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); } for (TransportRequest internalRequest : requests) { - assertThat(internalRequest, instanceOf(IndicesRequest.class)); - IndicesRequest indicesRequest = (IndicesRequest) internalRequest; + IndicesRequest indicesRequest = convertRequest(internalRequest); assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), equalTo(originalRequest.indices())); assertThat(indicesRequest.indicesOptions(), equalTo(originalRequest.indicesOptions())); } @@ -651,14 +650,24 @@ public class IndicesRequestIT extends ESIntegTestCase { List requests = consumeTransportRequests(action); assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); for (TransportRequest internalRequest : requests) { - assertThat(internalRequest, instanceOf(IndicesRequest.class)); - for (String index : ((IndicesRequest) internalRequest).indices()) { + IndicesRequest indicesRequest = convertRequest(internalRequest); + for (String index : indicesRequest.indices()) { assertThat(indices, hasItem(index)); } } } } + static IndicesRequest convertRequest(TransportRequest request) { + final IndicesRequest indicesRequest; + if (request instanceof IndicesRequest) { + indicesRequest = (IndicesRequest) request; + } else { + indicesRequest = TransportReplicationActionTests.resolveRequest(request); + } + return indicesRequest; + } + private String randomIndexOrAlias() { String index = randomFrom(indices); if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 05e32bbfdde..03cf86397c2 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -60,14 +60,6 @@ import static org.hamcrest.Matchers.nullValue; public class BulkWithUpdatesIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("script.default_lang", CustomScriptPlugin.NAME) - .build(); - } - @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -557,6 +549,7 @@ public class BulkWithUpdatesIT extends ESIntegTestCase { " \"script\" : {" + " \"inline\" : \"ctx._source.field2 = 'value2'\"" + " }," + + " \"lang\" : \"" + CustomScriptPlugin.NAME + "\"," + " \"upsert\" : {" + " \"field1\" : \"value1'\"" + " }" + @@ -589,7 +582,9 @@ public class BulkWithUpdatesIT extends ESIntegTestCase { assertThat(bulkResponse.getItems().length, equalTo(3)); assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false)); assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false)); - assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(false)); + assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(true)); + assertThat(bulkResponse.getItems()[2].getFailure().getCause().getCause().getMessage(), + equalTo("script_lang not supported [painless]")); client().admin().indices().prepareRefresh("test").get(); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 6c30f015124..bda117642a0 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -36,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; @@ -47,21 +49,25 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardClosedException; +import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseOptions; import org.elasticsearch.transport.TransportService; @@ -75,12 +81,12 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import java.util.stream.Collectors; import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state; @@ -93,12 +99,32 @@ import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TransportReplicationActionTests extends ESTestCase { + /** + * takes a request that was sent by a {@link TransportReplicationAction} and captured + * and returns the underlying request if it's wrapped or the original (cast to the expected type). + * + * This will throw a {@link ClassCastException} if the request is of the wrong type. + */ + public static R resolveRequest(TransportRequest requestOrWrappedRequest) { + if (requestOrWrappedRequest instanceof TransportReplicationAction.ConcreteShardRequest) { + requestOrWrappedRequest = ((TransportReplicationAction.ConcreteShardRequest)requestOrWrappedRequest).getRequest(); + } + return (R) requestOrWrappedRequest; + } + private static ThreadPool threadPool; private ClusterService clusterService; @@ -411,7 +437,7 @@ public class TransportReplicationActionTests extends ESTestCase { isRelocated.set(true); executeOnPrimary = false; } - action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { + action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -452,7 +478,8 @@ public class TransportReplicationActionTests extends ESTestCase { final String index = "test"; final ShardId shardId = new ShardId(index, "_na_", 0); ClusterState state = state(index, true, ShardRoutingState.RELOCATING); - String primaryTargetNodeId = state.getRoutingTable().shardRoutingTable(shardId).primaryShard().relocatingNodeId(); + final ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard(); + String primaryTargetNodeId = primaryShard.relocatingNodeId(); // simulate execution of the primary phase on the relocation target node state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryTargetNodeId)).build(); setState(clusterService, state); @@ -460,7 +487,7 @@ public class TransportReplicationActionTests extends ESTestCase { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); AtomicBoolean executed = new AtomicBoolean(); - action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { + action.new AsyncPrimaryAction(request, primaryShard.allocationId().getRelocationId(), createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -473,6 +500,11 @@ public class TransportReplicationActionTests extends ESTestCase { } }; } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } }.run(); assertThat(executed.get(), equalTo(true)); assertPhase(task, "finished"); @@ -596,7 +628,9 @@ public class TransportReplicationActionTests extends ESTestCase { state = ClusterState.builder(state).metaData(metaData).build(); setState(clusterService, state); AtomicBoolean executed = new AtomicBoolean(); - action.new AsyncPrimaryAction(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null) { + ShardRouting primaryShard = state.routingTable().shardRoutingTable(shardId).primaryShard(); + action.new AsyncPrimaryAction(new Request(shardId), primaryShard.allocationId().getId(), + createTransportChannel(new PlainActionFuture<>()), null) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -613,8 +647,10 @@ public class TransportReplicationActionTests extends ESTestCase { final String index = "test"; final ShardId shardId = new ShardId(index, "_na_", 0); // no replica, we only want to test on primary - setState(clusterService, state(index, true, ShardRoutingState.STARTED)); + final ClusterState state = state(index, true, ShardRoutingState.STARTED); + setState(clusterService, state); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); + final ShardRouting primaryShard = state.routingTable().shardRoutingTable(shardId).primaryShard(); Request request = new Request(shardId); PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); @@ -622,7 +658,7 @@ public class TransportReplicationActionTests extends ESTestCase { final boolean throwExceptionOnCreation = i == 1; final boolean throwExceptionOnRun = i == 2; final boolean respondWithError = i == 3; - action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { + action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -666,8 +702,9 @@ public class TransportReplicationActionTests extends ESTestCase { public void testReplicasCounter() throws Exception { final ShardId shardId = new ShardId("test", "_na_", 0); - setState(clusterService, state(shardId.getIndexName(), true, - ShardRoutingState.STARTED, ShardRoutingState.STARTED)); + final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED); + setState(clusterService, state); + final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0); boolean throwException = randomBoolean(); final ReplicationTask task = maybeTask(); Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) { @@ -683,7 +720,9 @@ public class TransportReplicationActionTests extends ESTestCase { }; final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler(); try { - replicaOperationTransportHandler.messageReceived(new Request().setShardId(shardId), + replicaOperationTransportHandler.messageReceived( + new TransportReplicationAction.ConcreteShardRequest<>( + new Request().setShardId(shardId), replicaRouting.allocationId().getId()), createTransportChannel(new PlainActionFuture<>()), task); } catch (ElasticsearchException e) { assertThat(e.getMessage(), containsString("simulated")); @@ -725,6 +764,111 @@ public class TransportReplicationActionTests extends ESTestCase { assertEquals(ActiveShardCount.from(requestWaitForActiveShards), request.waitForActiveShards()); } + /** test that a primary request is rejected if it arrives at a shard with a wrong allocation id */ + public void testPrimaryActionRejectsWrongAid() throws Exception { + final String index = "test"; + final ShardId shardId = new ShardId(index, "_na_", 0); + setState(clusterService, state(index, true, ShardRoutingState.STARTED)); + PlainActionFuture listener = new PlainActionFuture<>(); + Request request = new Request(shardId).timeout("1ms"); + action.new PrimaryOperationTransportHandler().messageReceived( + new TransportReplicationAction.ConcreteShardRequest<>(request, "_not_a_valid_aid_"), + createTransportChannel(listener), maybeTask() + ); + try { + listener.get(); + fail("using a wrong aid didn't fail the operation"); + } catch (ExecutionException execException) { + Throwable throwable = execException.getCause(); + logger.debug("got exception:" , throwable); + assertTrue(throwable.getClass() + " is not a retry exception", action.retryPrimaryException(throwable)); + } + } + + /** test that a replica request is rejected if it arrives at a shard with a wrong allocation id */ + public void testReplicaActionRejectsWrongAid() throws Exception { + final String index = "test"; + final ShardId shardId = new ShardId(index, "_na_", 0); + ClusterState state = state(index, false, ShardRoutingState.STARTED, ShardRoutingState.STARTED); + final ShardRouting replica = state.routingTable().shardRoutingTable(shardId).replicaShards().get(0); + // simulate execution of the node holding the replica + state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(replica.currentNodeId())).build(); + setState(clusterService, state); + + PlainActionFuture listener = new PlainActionFuture<>(); + Request request = new Request(shardId).timeout("1ms"); + action.new ReplicaOperationTransportHandler().messageReceived( + new TransportReplicationAction.ConcreteShardRequest<>(request, "_not_a_valid_aid_"), + createTransportChannel(listener), maybeTask() + ); + try { + listener.get(); + fail("using a wrong aid didn't fail the operation"); + } catch (ExecutionException execException) { + Throwable throwable = execException.getCause(); + if (action.retryPrimaryException(throwable) == false) { + throw new AssertionError("thrown exception is not retriable", throwable); + } + assertThat(throwable.getMessage(), containsString("_not_a_valid_aid_")); + } + } + + /** + * test throwing a {@link org.elasticsearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException} + * causes a retry + */ + public void testRetryOnReplica() throws Exception { + final ShardId shardId = new ShardId("test", "_na_", 0); + ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED); + final ShardRouting replica = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0); + // simulate execution of the node holding the replica + state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(replica.currentNodeId())).build(); + setState(clusterService, state); + AtomicBoolean throwException = new AtomicBoolean(true); + final ReplicationTask task = maybeTask(); + Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) { + @Override + protected ReplicaResult shardOperationOnReplica(Request request) { + assertPhase(task, "replica"); + if (throwException.get()) { + throw new RetryOnReplicaException(shardId, "simulation"); + } + return new ReplicaResult(); + } + }; + final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler(); + final PlainActionFuture listener = new PlainActionFuture<>(); + final Request request = new Request().setShardId(shardId); + request.primaryTerm(state.metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id())); + replicaOperationTransportHandler.messageReceived( + new TransportReplicationAction.ConcreteShardRequest<>(request, replica.allocationId().getId()), + createTransportChannel(listener), task); + if (listener.isDone()) { + listener.get(); // fail with the exception if there + fail("listener shouldn't be done"); + } + + // no retry yet + List capturedRequests = + transport.getCapturedRequestsByTargetNodeAndClear().get(replica.currentNodeId()); + assertThat(capturedRequests, nullValue()); + + // release the waiting + throwException.set(false); + setState(clusterService, state); + + capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear().get(replica.currentNodeId()); + assertThat(capturedRequests, notNullValue()); + assertThat(capturedRequests.size(), equalTo(1)); + final CapturingTransport.CapturedRequest capturedRequest = capturedRequests.get(0); + assertThat(capturedRequest.action, equalTo("testActionWithExceptions[r]")); + assertThat(capturedRequest.request, instanceOf(TransportReplicationAction.ConcreteShardRequest.class)); + assertThat(((TransportReplicationAction.ConcreteShardRequest) capturedRequest.request).getRequest(), equalTo(request)); + assertThat(((TransportReplicationAction.ConcreteShardRequest) capturedRequest.request).getTargetAllocationID(), + equalTo(replica.allocationId().getId())); + } + + private void assertIndexShardCounter(int expected) { assertThat(count.get(), equalTo(expected)); } @@ -797,7 +941,7 @@ public class TransportReplicationActionTests extends ESTestCase { Action(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { - super(settings, actionName, transportService, clusterService, null, threadPool, + super(settings, actionName, transportService, clusterService, mockIndicesService(clusterService), threadPool, new ShardStateAction(settings, clusterService, transportService, null, null, threadPool), new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY), Request::new, Request::new, ThreadPool.Names.SAME); @@ -825,43 +969,76 @@ public class TransportReplicationActionTests extends ESTestCase { protected boolean resolveIndex() { return false; } + } - @Override - protected void acquirePrimaryShardReference(ShardId shardId, ActionListener onReferenceAcquired) { + final IndicesService mockIndicesService(ClusterService clusterService) { + final IndicesService indicesService = mock(IndicesService.class); + when(indicesService.indexServiceSafe(any(Index.class))).then(invocation -> { + Index index = (Index)invocation.getArguments()[0]; + final ClusterState state = clusterService.state(); + final IndexMetaData indexSafe = state.metaData().getIndexSafe(index); + return mockIndexService(indexSafe, clusterService); + }); + when(indicesService.indexService(any(Index.class))).then(invocation -> { + Index index = (Index) invocation.getArguments()[0]; + final ClusterState state = clusterService.state(); + if (state.metaData().hasIndex(index.getName())) { + final IndexMetaData indexSafe = state.metaData().getIndexSafe(index); + return mockIndexService(clusterService.state().metaData().getIndexSafe(index), clusterService); + } else { + return null; + } + }); + return indicesService; + } + + final IndexService mockIndexService(final IndexMetaData indexMetaData, ClusterService clusterService) { + final IndexService indexService = mock(IndexService.class); + when(indexService.getShard(anyInt())).then(invocation -> { + int shard = (Integer) invocation.getArguments()[0]; + final ShardId shardId = new ShardId(indexMetaData.getIndex(), shard); + if (shard > indexMetaData.getNumberOfShards()) { + throw new ShardNotFoundException(shardId); + } + return mockIndexShard(shardId, clusterService); + }); + return indexService; + } + + private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService) { + final IndexShard indexShard = mock(IndexShard.class); + doAnswer(invocation -> { + ActionListener callback = (ActionListener) invocation.getArguments()[0]; count.incrementAndGet(); - PrimaryShardReference primaryShardReference = new PrimaryShardReference(null, null) { - @Override - public boolean isRelocated() { - return isRelocated.get(); - } - - @Override - public void failShard(String reason, @Nullable Exception e) { - throw new UnsupportedOperationException(); - } - - @Override - public ShardRouting routingEntry() { - ShardRouting shardRouting = clusterService.state().getRoutingTable() - .shardRoutingTable(shardId).primaryShard(); - assert shardRouting != null; - return shardRouting; - } - - @Override - public void close() { - count.decrementAndGet(); - } - }; - - onReferenceAcquired.onResponse(primaryShardReference); - } - - @Override - protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, ActionListener onLockAcquired) { + callback.onResponse(count::decrementAndGet); + return null; + }).when(indexShard).acquirePrimaryOperationLock(any(ActionListener.class), anyString()); + doAnswer(invocation -> { + long term = (Long)invocation.getArguments()[0]; + ActionListener callback = (ActionListener) invocation.getArguments()[1]; + final long primaryTerm = indexShard.getPrimaryTerm(); + if (term < primaryTerm) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])", + shardId, term, primaryTerm)); + } count.incrementAndGet(); - onLockAcquired.onResponse(count::decrementAndGet); - } + callback.onResponse(count::decrementAndGet); + return null; + }).when(indexShard).acquireReplicaOperationLock(anyLong(), any(ActionListener.class), anyString()); + when(indexShard.routingEntry()).thenAnswer(invocationOnMock -> { + final ClusterState state = clusterService.state(); + final RoutingNode node = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + final ShardRouting routing = node.getByShardId(shardId); + if (routing == null) { + throw new ShardNotFoundException(shardId, "shard is no longer assigned to current node"); + } + return routing; + }); + when(indexShard.state()).thenAnswer(invocationOnMock -> isRelocated.get() ? IndexShardState.RELOCATED : IndexShardState.STARTED); + doThrow(new AssertionError("failed shard is not supported")).when(indexShard).failShard(anyString(), any(Exception.class)); + when(indexShard.getPrimaryTerm()).thenAnswer(i -> + clusterService.state().metaData().getIndexSafe(shardId.getIndex()).primaryTerm(shardId.id())); + return indexShard; } class NoopReplicationOperation extends ReplicationOperation { @@ -879,11 +1056,6 @@ public class TransportReplicationActionTests extends ESTestCase { * Transport channel that is needed for replica operation testing. */ public TransportChannel createTransportChannel(final PlainActionFuture listener) { - return createTransportChannel(listener, error -> { - }); - } - - public TransportChannel createTransportChannel(final PlainActionFuture listener, Consumer consumer) { return new TransportChannel() { @Override @@ -908,7 +1080,6 @@ public class TransportReplicationActionTests extends ESTestCase { @Override public void sendResponse(Exception exception) throws IOException { - consumer.accept(exception); listener.onFailure(exception); } diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index a98433a1007..7b606ee4159 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -55,7 +55,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); Map params = script.getParams(); assertThat(params, nullValue()); @@ -67,7 +67,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, nullValue()); @@ -79,7 +79,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); @@ -92,7 +92,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); @@ -107,7 +107,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); @@ -124,7 +124,7 @@ public class UpdateRequestTests extends ESTestCase { assertThat(script, notNullValue()); assertThat(script.getScript(), equalTo("script1")); assertThat(script.getType(), equalTo(ScriptType.INLINE)); - assertThat(script.getLang(), nullValue()); + assertThat(script.getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); params = script.getParams(); assertThat(params, notNullValue()); assertThat(params.size(), equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java index bb3ef29176e..467608922d1 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java @@ -111,15 +111,9 @@ public class JarHellTests extends ESTestCase { } } - public void testLog4jLeniency() throws Exception { + public void testLog4jThrowableProxyLeniency() throws Exception { Path dir = createTempDir(); - URL[] jars = {makeJar(dir, "foo.jar", null, "org/apache/log4j/DuplicateClass.class"), makeJar(dir, "bar.jar", null, "org/apache/log4j/DuplicateClass.class")}; - JarHell.checkJarHell(jars); - } - - public void testBaseDateTimeLeniency() throws Exception { - Path dir = createTempDir(); - URL[] jars = {makeJar(dir, "foo.jar", null, "org/joda/time/base/BaseDateTime.class"), makeJar(dir, "bar.jar", null, "org/joda/time/base/BaseDateTime.class")}; + URL[] jars = {makeJar(dir, "foo.jar", null, "org.apache.logging.log4j.core.impl.ThrowableProxy.class"), makeJar(dir, "bar.jar", null, "org.apache.logging.log4j.core.impl.ThrowableProxy.class")}; JarHell.checkJarHell(jars); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 75c33c44cf1..98268e409fc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -56,6 +56,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; @@ -729,10 +730,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ImmutableOpenMap shardSizes = shardSizesBuilder.build(); final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes); + DiskThresholdDecider decider = makeDecider(diskSettings); AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, - new HashSet<>(Arrays.asList( - new SameShardAllocationDecider(Settings.EMPTY), - makeDecider(diskSettings)))); + new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY), decider))); ClusterInfoService cis = new ClusterInfoService() { @Override @@ -832,6 +832,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder(); shardSizesBuilder.put("[test][0][p]", 40L); shardSizesBuilder.put("[test][1][p]", 40L); + shardSizesBuilder.put("[foo][0][p]", 10L); ImmutableOpenMap shardSizes = shardSizesBuilder.build(); final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes); @@ -839,10 +840,12 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .put(IndexMetaData.builder("foo").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)) .build(); RoutingTable initialRoutingTable = RoutingTable.builder() .addAsNew(metaData.index("test")) + .addAsNew(metaData.index("foo")) .build(); DiscoveryNode discoveryNode1 = new DiscoveryNode("node1", new LocalTransportAddress("1"), emptyMap(), @@ -881,6 +884,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED); secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", true, ShardRoutingState.RELOCATING); + ShardRouting fooRouting = TestShardRouting.newShardRouting("foo", 0, "node1", null, true, ShardRoutingState.UNASSIGNED); firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting); builder = RoutingTable.builder().add( IndexRoutingTable.builder(firstRouting.index()) @@ -898,6 +902,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { false); decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); + decision = diskThresholdDecider.canAllocate(fooRouting, firstRoutingNode, routingAllocation); + assertThat(decision.type(), equalTo(Decision.Type.NO)); // Creating AllocationService instance and the services it depends on... ClusterInfoService cis = new ClusterInfoService() { diff --git a/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java new file mode 100644 index 00000000000..f6d411c0df3 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/settings/MemorySizeSettingsTests.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.indices.IndexingMemoryController; +import org.elasticsearch.indices.IndicesQueryCache; +import org.elasticsearch.indices.IndicesRequestCache; +import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; + +public class MemorySizeSettingsTests extends ESTestCase { + + public void testPageCacheLimitHeapSetting() { + assertMemorySizeSetting(PageCacheRecycler.LIMIT_HEAP_SETTING, "cache.recycler.page.limit.heap", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1))); + } + + public void testIndexBufferSizeSetting() { + assertMemorySizeSetting(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "indices.memory.index_buffer_size", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1))); + } + + public void testQueryCacheSizeSetting() { + assertMemorySizeSetting(IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, "indices.queries.cache.size", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1))); + } + + public void testIndicesRequestCacheSetting() { + assertMemorySizeSetting(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE, "indices.requests.cache.size", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.01))); + } + + public void testCircuitBreakerSettings() { + assertMemorySizeSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.total.limit", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.7))); + assertMemorySizeSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.fielddata.limit", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6))); + assertMemorySizeSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.request.limit", + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6))); + assertMemorySizeSetting(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING, + "network.breaker.inflight_requests.limit", new ByteSizeValue((JvmInfo.jvmInfo().getMem().getHeapMax().bytes()))); + } + + public void testIndicesFieldDataCacheSetting() { + assertMemorySizeSetting(IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY, "indices.fielddata.cache.size", + new ByteSizeValue(-1)); + } + + private void assertMemorySizeSetting(Setting setting, String settingKey, ByteSizeValue defaultValue) { + assertThat(setting, notNullValue()); + assertThat(setting.getKey(), equalTo(settingKey)); + assertThat(setting.getProperties(), hasItem(Property.NodeScope)); + assertThat(setting.getDefault(Settings.EMPTY), + equalTo(defaultValue)); + Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build(); + assertThat(setting.get(settingWithPercentage), + equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25)))); + Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build(); + assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024))); + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java index b1504672f4e..17bd0d93d72 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.test.ESTestCase; import java.util.Arrays; @@ -68,6 +69,44 @@ public class SettingTests extends ESTestCase { assertEquals(new ByteSizeValue(12), value.get()); } + public void testMemorySize() { + Setting memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic, + Property.NodeScope); + + assertFalse(memorySizeValueSetting.isGroupSetting()); + ByteSizeValue memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY); + assertEquals(memorySizeValue.bytes(), 1024); + + memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope); + memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY); + assertEquals(memorySizeValue.bytes(), 2048); + + memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", "50%", Property.Dynamic, Property.NodeScope); + assertFalse(memorySizeValueSetting.isGroupSetting()); + memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY); + assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.5, 1.0); + + memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "25%", Property.Dynamic, Property.NodeScope); + memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY); + assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25, 1.0); + + AtomicReference value = new AtomicReference<>(null); + ClusterSettings.SettingUpdater settingUpdater = memorySizeValueSetting.newUpdater(value::set, logger); + try { + settingUpdater.apply(Settings.builder().put("a.byte.size", 12).build(), Settings.EMPTY); + fail("no unit"); + } catch (IllegalArgumentException ex) { + assertEquals("failed to parse setting [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized", + ex.getMessage()); + } + + assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12b").build(), Settings.EMPTY)); + assertEquals(new ByteSizeValue(12), value.get()); + + assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY)); + assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.2)), value.get()); + } + public void testSimpleUpdate() { Setting booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope); AtomicReference atomicBoolean = new AtomicReference<>(null); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java new file mode 100644 index 00000000000..60f01cac700 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/analysis/MinHashFilterFactoryTests.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTokenStreamTestCase; + +import java.io.IOException; +import java.io.StringReader; + +public class MinHashFilterFactoryTests extends ESTokenStreamTestCase { + public void testDefault() throws IOException { + int default_hash_count = 1; + int default_bucket_size = 512; + int default_hash_set_size = 1; + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); + TokenFilterFactory tokenFilter = analysisService.tokenFilter("min_hash"); + String source = "the quick brown fox"; + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader(source)); + + // with_rotation is true by default, and hash_set_size is 1, so even though the source doesn't + // have enough tokens to fill all the buckets, we still expect 512 tokens. + assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), + default_hash_count * default_bucket_size * default_hash_set_size); + } + + public void testSettings() throws IOException { + Settings settings = Settings.builder() + .put("index.analysis.filter.test_min_hash.type", "min_hash") + .put("index.analysis.filter.test_min_hash.hash_count", "1") + .put("index.analysis.filter.test_min_hash.bucket_count", "2") + .put("index.analysis.filter.test_min_hash.hash_set_size", "1") + .put("index.analysis.filter.test_min_hash.with_rotation", false) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); + TokenFilterFactory tokenFilter = analysisService.tokenFilter("test_min_hash"); + String source = "sushi"; + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader(source)); + + // despite the fact that bucket_count is 2 and hash_set_size is 1, + // because with_rotation is false, we only expect 1 token here. + assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), 1); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java new file mode 100644 index 00000000000..a39fbae1764 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.ScriptSettings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; + +import static java.util.Collections.emptyList; + +public class QueryRewriteContextTests extends ESTestCase { + + public void testNewParseContextWithLegacyScriptLanguage() throws Exception { + String defaultLegacyScriptLanguage = randomAsciiOfLength(4); + IndexMetaData.Builder indexMetadata = new IndexMetaData.Builder("index"); + indexMetadata.settings(Settings.builder().put("index.version.created", Version.CURRENT) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 1) + ); + IndicesQueriesRegistry indicesQueriesRegistry = new SearchModule(Settings.EMPTY, false, emptyList()).getQueryParserRegistry(); + IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), + Settings.builder().put(ScriptSettings.LEGACY_SCRIPT_SETTING, defaultLegacyScriptLanguage).build()); + QueryRewriteContext queryRewriteContext = + new QueryRewriteContext(indexSettings, null, null, indicesQueriesRegistry, null, null, null);; + + // verify that the default script language in the query parse context is equal to defaultLegacyScriptLanguage variable: + QueryParseContext queryParseContext = + queryRewriteContext.newParseContextWithLegacyScriptLanguage(XContentHelper.createParser(new BytesArray("{}"))); + assertEquals(defaultLegacyScriptLanguage, queryParseContext.getDefaultScriptLanguage()); + + // verify that the script query's script language is equal to defaultLegacyScriptLanguage variable: + XContentParser parser = XContentHelper.createParser(new BytesArray("{\"script\" : {\"script\": \"return true\"}}")); + queryParseContext = queryRewriteContext.newParseContextWithLegacyScriptLanguage(parser); + ScriptQueryBuilder queryBuilder = (ScriptQueryBuilder) queryParseContext.parseInnerQueryBuilder().get(); + assertEquals(defaultLegacyScriptLanguage, queryBuilder.script().getLang()); + } + +} diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index d85b32145ea..71c96b85fd5 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.Arrays; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -105,7 +106,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("--> refreshing the index"); refreshAndAssert(); logger.info("--> verifying indexed content"); - iterateAssertCount(numberOfShards, indexer.totalIndexedDocs(), 10); + iterateAssertCount(numberOfShards, 10, indexer.getIds()); } } @@ -156,7 +157,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("--> refreshing the index"); refreshAndAssert(); logger.info("--> verifying indexed content"); - iterateAssertCount(numberOfShards, indexer.totalIndexedDocs(), 10); + iterateAssertCount(numberOfShards, 10, indexer.getIds()); } } @@ -225,7 +226,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("--> refreshing the index"); refreshAndAssert(); logger.info("--> verifying indexed content"); - iterateAssertCount(numberOfShards, indexer.totalIndexedDocs(), 10); + iterateAssertCount(numberOfShards, 10, indexer.getIds()); } } @@ -263,11 +264,12 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("--> refreshing the index"); refreshAndAssert(); logger.info("--> verifying indexed content"); - iterateAssertCount(numShards, indexer.totalIndexedDocs(), 10); + iterateAssertCount(numShards, 10, indexer.getIds()); } } - private void iterateAssertCount(final int numberOfShards, final long numberOfDocs, final int iterations) throws Exception { + private void iterateAssertCount(final int numberOfShards, final int iterations, final Set ids) throws Exception { + final long numberOfDocs = ids.size(); SearchResponse[] iterationResults = new SearchResponse[iterations]; boolean error = false; for (int i = 0; i < iterations; i++) { @@ -290,12 +292,11 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { ClusterService clusterService = clusterService(); final ClusterState state = clusterService.state(); for (int shard = 0; shard < numberOfShards; shard++) { - // background indexer starts using ids on 1 - for (int id = 1; id <= numberOfDocs; id++) { - ShardId docShard = clusterService.operationRouting().shardId(state, "test", Long.toString(id), null); + for (String id : ids) { + ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null); if (docShard.id() == shard) { for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) { - GetResponse response = client().prepareGet("test", "type", Long.toString(id)) + GetResponse response = client().prepareGet("test", "type", id) .setPreference("_only_nodes:" + shardRouting.currentNodeId()).get(); if (response.isExists()) { logger.info("missing id [{}] on shard {}", id, shardRouting); @@ -321,6 +322,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { TimeUnit.MINUTES ) ); + assertEquals(numberOfDocs, ids.size()); } //lets now make the test fail if it was supposed to fail diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 02c37e2ad55..8493a08d704 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -232,13 +232,8 @@ public class RelocationIT extends ESIntegTestCase { logger.error("Extra id [{}]", id); } } - set.forEach(new IntProcedure() { - - @Override - public void apply(int value) { - logger.error("Missing id [{}]", value); - } - + set.forEach((IntProcedure) value -> { + logger.error("Missing id [{}]", value); }); } assertThat(hits.totalHits(), equalTo(indexer.totalIndexedDocs())); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 3e07d3c170d..7b345b137b9 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -45,6 +45,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; @@ -86,7 +87,9 @@ public class ScriptServiceTests extends ESTestCase { resourceWatcherService = new ResourceWatcherService(baseSettings, null); scriptEngineService = new TestEngineService(); dangerousScriptEngineService = new TestDangerousEngineService(); - scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap(Collections.singleton(scriptEngineService)); + TestEngineService defaultScriptServiceEngine = new TestEngineService(Script.DEFAULT_SCRIPT_LANG) {}; + scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap( + new HashSet<>(Arrays.asList(scriptEngineService, defaultScriptServiceEngine))); //randomly register custom script contexts int randomInt = randomIntBetween(0, 3); //prevent duplicates using map @@ -103,7 +106,8 @@ public class ScriptServiceTests extends ESTestCase { String context = plugin + "_" + operation; contexts.put(context, new ScriptContext.Plugin(plugin, operation)); } - scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineService, dangerousScriptEngineService)); + scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineService, dangerousScriptEngineService, + defaultScriptServiceEngine)); scriptContextRegistry = new ScriptContextRegistry(contexts.values()); scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); @@ -406,12 +410,11 @@ public class ScriptServiceTests extends ESTestCase { public void testDefaultLanguage() throws IOException { Settings.Builder builder = Settings.builder(); - builder.put("script.default_lang", "test"); builder.put("script.inline", "true"); buildScriptService(builder.build()); CompiledScript script = scriptService.compile(new Script("1 + 1", ScriptType.INLINE, null, null), randomFrom(scriptContexts), Collections.emptyMap()); - assertEquals(script.lang(), "test"); + assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG); } public void testStoreScript() throws Exception { @@ -509,14 +512,24 @@ public class ScriptServiceTests extends ESTestCase { public static final String NAME = "test"; + private final String name; + + public TestEngineService() { + this(NAME); + } + + public TestEngineService(String name) { + this.name = name; + } + @Override public String getType() { - return NAME; + return name; } @Override public String getExtension() { - return NAME; + return name; } @Override diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java index 120fd38b9a0..917650d36b8 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java @@ -34,32 +34,33 @@ import static org.hamcrest.Matchers.equalTo; public class ScriptSettingsTests extends ESTestCase { - public void testDefaultLanguageIsPainless() { + public void testDefaultLegacyLanguageIsPainless() { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService())); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(Settings.EMPTY), equalTo("painless")); + assertThat(scriptSettings.getDefaultLegacyScriptLanguageSetting().get(Settings.EMPTY), + equalTo(ScriptSettings.LEGACY_DEFAULT_LANG)); } - public void testCustomDefaultLanguage() { + public void testCustomLegacyDefaultLanguage() { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService())); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); String defaultLanguage = CustomScriptEngineService.NAME; - Settings settings = Settings.builder().put("script.default_lang", defaultLanguage).build(); - assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(settings), equalTo(defaultLanguage)); + Settings settings = Settings.builder().put(ScriptSettings.LEGACY_SCRIPT_SETTING, defaultLanguage).build(); + assertThat(scriptSettings.getDefaultLegacyScriptLanguageSetting().get(settings), equalTo(defaultLanguage)); } - public void testInvalidDefaultLanguage() { + public void testInvalidLegacyDefaultLanguage() { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService())); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - Settings settings = Settings.builder().put("script.default_lang", "C++").build(); + Settings settings = Settings.builder().put(ScriptSettings.LEGACY_SCRIPT_SETTING, "C++").build(); try { - scriptSettings.getDefaultScriptLanguageSetting().get(settings); + scriptSettings.getDefaultLegacyScriptLanguageSetting().get(settings); fail("should have seen unregistered default language"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("unregistered default language [C++]")); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index 2d9d5ca043d..fab1f8b7d3e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -20,12 +20,10 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.plugins.Plugin; @@ -49,6 +47,7 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Signi import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.search.aggregations.bucket.SharedSignificantTermsTestMethods; @@ -172,7 +171,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase { @Override public List> getSignificanceHeuristics() { return singletonList(new SearchExtensionSpec(SimpleHeuristic.NAME, - SimpleHeuristic::new, SimpleHeuristic::parse)); + SimpleHeuristic::new, (context) -> SimpleHeuristic.parse(context))); } @Override @@ -239,9 +238,9 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase { return subsetFreq / subsetSize > supersetFreq / supersetSize ? 2.0 : 1.0; } - public static SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + public static SignificanceHeuristic parse(XContentParseContext context) throws IOException, QueryShardException { - parser.nextToken(); + context.getParser().nextToken(); return new SimpleHeuristic(); } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java index 6a4cc61e041..e96b02c0e2c 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java @@ -182,7 +182,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase println "Indexed $response.id into $response.index/$response.type" @@ -65,7 +65,7 @@ This option allows to pass the actual instance of the request (instead of a closure) as a parameter. The rest is similar to the closure as a parameter option (the `GActionFuture` handling). For example: -[source,js] +[source,groovy] -------------------------------------------------- def indexR = client.index (new IndexRequest( index: "test", @@ -90,7 +90,7 @@ The last option is to provide an actual instance of the API request, and an `ActionListener` for the callback. This is exactly like the Java API with the added `gexecute` which returns the `GActionFuture`: -[source,js] +[source,groovy] -------------------------------------------------- def indexR = node.client.prepareIndex("test", "type1", "1").setSource({ test = "value" diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc index c0a6d688415..a2745f459bd 100644 --- a/docs/groovy-api/client.asciidoc +++ b/docs/groovy-api/client.asciidoc @@ -13,7 +13,7 @@ within the cluster. A Node based client is the simplest form to get a `GClient` to start executing operations against elasticsearch. -[source,js] +[source,groovy] -------------------------------------------------- import org.elasticsearch.groovy.client.GClient import org.elasticsearch.groovy.node.GNode @@ -33,7 +33,7 @@ Since elasticsearch allows to configure it using JSON based settings, the configuration itself can be done using a closure that represent the JSON: -[source,js] +[source,groovy] -------------------------------------------------- import org.elasticsearch.groovy.node.GNode import org.elasticsearch.groovy.node.GNodeBuilder diff --git a/docs/groovy-api/delete.asciidoc b/docs/groovy-api/delete.asciidoc index e3320126966..18f11e67775 100644 --- a/docs/groovy-api/delete.asciidoc +++ b/docs/groovy-api/delete.asciidoc @@ -6,7 +6,7 @@ The delete API is very similar to the Java delete API, here is an example: -[source,js] +[source,groovy] -------------------------------------------------- def deleteF = node.client.delete { index "test" diff --git a/docs/groovy-api/get.asciidoc b/docs/groovy-api/get.asciidoc index 6bf476c16a8..824c18f8f11 100644 --- a/docs/groovy-api/get.asciidoc +++ b/docs/groovy-api/get.asciidoc @@ -7,7 +7,7 @@ Java get API. The main benefit of using groovy is handling the source content. It can be automatically converted to a `Map` which means using Groovy to navigate it is simple: -[source,js] +[source,groovy] -------------------------------------------------- def getF = node.client.get { index "test" diff --git a/docs/groovy-api/index_.asciidoc b/docs/groovy-api/index_.asciidoc index cd7f0ca4ac9..b63a212352a 100644 --- a/docs/groovy-api/index_.asciidoc +++ b/docs/groovy-api/index_.asciidoc @@ -7,7 +7,7 @@ Java index API. The Groovy extension to it is the ability to provide the indexed source using a closure. For example: -[source,js] +[source,groovy] -------------------------------------------------- def indexR = client.index { index "test" diff --git a/docs/groovy-api/search.asciidoc b/docs/groovy-api/search.asciidoc index 946760d95cc..d0b74a4d658 100644 --- a/docs/groovy-api/search.asciidoc +++ b/docs/groovy-api/search.asciidoc @@ -7,7 +7,7 @@ Java search API. The Groovy extension allows to provide the search source to execute as a `Closure` including the query itself (similar to GORM criteria builder): -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.search { indices "test" @@ -19,7 +19,7 @@ def search = node.client.search { } } -search.response.hits.each {SearchHit hit -> +search.response.hits.each {SearchHit hit -> println "Got hit $hit.id from $hit.index/$hit.type" } -------------------------------------------------- @@ -27,13 +27,13 @@ search.response.hits.each {SearchHit hit -> It can also be executed using the "Java API" while still using a closure for the query: -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.prepareSearch("test").setQuery({ term(test: "value") }).gexecute(); -search.response.hits.each {SearchHit hit -> +search.response.hits.each {SearchHit hit -> println "Got hit $hit.id from $hit.index/$hit.type" } -------------------------------------------------- @@ -48,7 +48,7 @@ The format of the search `Closure` follows the same JSON syntax as the Term query where multiple values are provided (see {ref}/query-dsl-terms-query.html[terms]): -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.search { indices "test" @@ -64,7 +64,7 @@ def search = node.client.search { Query string (see {ref}/query-dsl-query-string-query.html[query string]): -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.search { indices "test" @@ -82,7 +82,7 @@ def search = node.client.search { Pagination (see {ref}/search-request-from-size.html[from/size]): -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.search { indices "test" @@ -99,7 +99,7 @@ def search = node.client.search { Sorting (see {ref}/search-request-sort.html[sort]): -[source,js] +[source,groovy] -------------------------------------------------- def search = node.client.search { indices "test" diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index b43a10e25b9..a21e0c5c82f 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -17,7 +17,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install analysis-icu ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -32,7 +31,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove analysis-icu ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index 5899134cec0..90584645bbb 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -14,7 +14,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install analysis-kuromoji ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -29,7 +28,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove analysis-kuromoji ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc index 4fcfcf6caba..34f14abe3c5 100644 --- a/docs/plugins/analysis-phonetic.asciidoc +++ b/docs/plugins/analysis-phonetic.asciidoc @@ -15,7 +15,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install analysis-phonetic ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -30,7 +29,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove analysis-phonetic ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/analysis-smartcn.asciidoc b/docs/plugins/analysis-smartcn.asciidoc index 665ccbaf611..18a020cf40d 100644 --- a/docs/plugins/analysis-smartcn.asciidoc +++ b/docs/plugins/analysis-smartcn.asciidoc @@ -20,7 +20,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install analysis-smartcn ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -35,7 +34,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove analysis-smartcn ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/analysis-stempel.asciidoc b/docs/plugins/analysis-stempel.asciidoc index 8a42135a879..88b43a1a805 100644 --- a/docs/plugins/analysis-stempel.asciidoc +++ b/docs/plugins/analysis-stempel.asciidoc @@ -17,7 +17,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install analysis-stempel ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -32,7 +31,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove analysis-stempel ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc index a9ec9929b49..0feb5f7f8e5 100644 --- a/docs/plugins/discovery-azure-classic.asciidoc +++ b/docs/plugins/discovery-azure-classic.asciidoc @@ -17,7 +17,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install discovery-azure-classic ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -32,7 +31,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove discovery-azure-classic ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. @@ -359,7 +357,7 @@ ssh azure-elasticsearch-cluster.cloudapp.net Once connected, install Elasticsearch: -[source,sh] +["source","sh",subs="attributes,callouts"] ---- # Install Latest Java version # Read http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html for details @@ -372,36 +370,43 @@ sudo apt-get install oracle-java8-installer # sudo apt-get install openjdk-8-jre-headless # Download Elasticsearch -curl -s https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.0.0.deb -o elasticsearch-2.0.0.deb +curl -s https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{version}.deb -o elasticsearch-{version}.deb # Prepare Elasticsearch installation -sudo dpkg -i elasticsearch-2.0.0.deb +sudo dpkg -i elasticsearch-{version}.deb ---- Check that elasticsearch is running: -[source,sh] +[source,js] ---- -curl http://localhost:9200/ +GET / ---- +// CONSOLE This command should give you a JSON result: -[source,js] ----- +["source","js",subs="attributes,callouts"] +-------------------------------------------- { - "status" : 200, - "name" : "Living Colossus", + "name" : "Cp8oag6", + "cluster_name" : "elasticsearch", "version" : { - "number" : "2.0.0", - "build_hash" : "a46900e9c72c0a623d71b54016357d5f94c8ea32", - "build_timestamp" : "2014-02-12T16:18:34Z", + "number" : "{version}", + "build_hash" : "f27399d", + "build_date" : "2016-03-30T09:51:41.449Z", "build_snapshot" : false, - "lucene_version" : "5.1" + "lucene_version" : "{lucene_version}" }, "tagline" : "You Know, for Search" } ----- +-------------------------------------------- +// TESTRESPONSE[s/"name" : "Cp8oag6",/"name" : "$body.name",/] +// TESTRESPONSE[s/"cluster_name" : "elasticsearch",/"cluster_name" : "$body.cluster_name",/] +// TESTRESPONSE[s/"build_hash" : "f27399d",/"build_hash" : "$body.version.build_hash",/] +// TESTRESPONSE[s/"build_date" : "2016-03-30T09:51:41.449Z",/"build_date" : $body.version.build_date,/] +// TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/] +// So much s/// but at least we test that the layout is close to matching.... [[discovery-azure-classic-long-plugin]] ===== Install elasticsearch cloud azure plugin diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index b62c5484905..0803d0a4fcd 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -15,7 +15,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install discovery-ec2 ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -30,7 +29,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove discovery-ec2 ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. @@ -231,6 +229,7 @@ Management Console. It should look similar to this. "Version": "2012-10-17" } ---- +// NOTCONSOLE [[discovery-ec2-filtering]] ===== Filtering by Tags diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index d13ba266b66..f615fc9810e 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -13,7 +13,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install discovery-gce ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -28,7 +27,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove discovery-gce ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/index.asciidoc b/docs/plugins/index.asciidoc index 8a572d5ee3f..128b12f5dba 100644 --- a/docs/plugins/index.asciidoc +++ b/docs/plugins/index.asciidoc @@ -1,7 +1,9 @@ = Elasticsearch Plugins and Integrations -:ref: https://www.elastic.co/guide/en/elasticsearch/reference/master -:guide: https://www.elastic.co/guide +:ref: https://www.elastic.co/guide/en/elasticsearch/reference/master +:guide: https://www.elastic.co/guide +:version: 5.0.0-alpha5 +:lucene_version: 6.2.0 [[intro]] == Introduction to plugins @@ -62,5 +64,3 @@ include::integrations.asciidoc[] include::authors.asciidoc[] include::redirects.asciidoc[] - - diff --git a/docs/plugins/ingest-attachment.asciidoc b/docs/plugins/ingest-attachment.asciidoc index 3c72e514094..65add6ac927 100644 --- a/docs/plugins/ingest-attachment.asciidoc +++ b/docs/plugins/ingest-attachment.asciidoc @@ -21,7 +21,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install ingest-attachment ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -36,7 +35,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove ingest-attachment ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc index d6eced47eca..1626be6c8e6 100644 --- a/docs/plugins/ingest-geoip.asciidoc +++ b/docs/plugins/ingest-geoip.asciidoc @@ -21,7 +21,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install ingest-geoip ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -36,7 +35,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove ingest-geoip ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/ingest-user-agent.asciidoc b/docs/plugins/ingest-user-agent.asciidoc index 95997a34c10..29903224f39 100644 --- a/docs/plugins/ingest-user-agent.asciidoc +++ b/docs/plugins/ingest-user-agent.asciidoc @@ -16,7 +16,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install ingest-user-agent ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -31,7 +30,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove ingest-user-agent ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc index 650f42d9cab..0b8346f2a90 100644 --- a/docs/plugins/lang-javascript.asciidoc +++ b/docs/plugins/lang-javascript.asciidoc @@ -17,7 +17,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install lang-javascript ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -32,7 +31,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove lang-javascript ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. @@ -147,11 +145,10 @@ JavaScript: First, save this file as `config/scripts/my_script.js` on every node in the cluster: -[source,js] +[source,painless] ---- doc["num"].value * factor ---- -// NOTCONSOLE then use the script as follows: diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc index fcec532ea10..a642e7f65cc 100644 --- a/docs/plugins/lang-python.asciidoc +++ b/docs/plugins/lang-python.asciidoc @@ -16,7 +16,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install lang-python ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -31,7 +30,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove lang-python ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc index 1a294cee78a..0e21c6ab705 100644 --- a/docs/plugins/mapper-attachments.asciidoc +++ b/docs/plugins/mapper-attachments.asciidoc @@ -19,7 +19,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install mapper-attachments ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -34,7 +33,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove mapper-attachments ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/mapper-murmur3.asciidoc b/docs/plugins/mapper-murmur3.asciidoc index f81c226d1f0..28b7a2387ef 100644 --- a/docs/plugins/mapper-murmur3.asciidoc +++ b/docs/plugins/mapper-murmur3.asciidoc @@ -15,7 +15,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install mapper-murmur3 ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -30,7 +29,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove mapper-murmur3 ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc index 4b2d02a6a2b..3c0277895d2 100644 --- a/docs/plugins/mapper-size.asciidoc +++ b/docs/plugins/mapper-size.asciidoc @@ -15,7 +15,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install mapper-size ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -30,7 +29,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove mapper-size ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 6c39975560f..f8fb8814129 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -15,7 +15,6 @@ Run the following command to get usage instructions: ----------------------------------- sudo bin/elasticsearch-plugin -h ----------------------------------- -// NOTCONSOLE [IMPORTANT] .Running as root @@ -42,7 +41,6 @@ Core Elasticsearch plugins can be installed as follows: ----------------------------------- sudo bin/elasticsearch-plugin install [plugin_name] ----------------------------------- -// NOTCONSOLE For instance, to install the core <>, just run the following command: @@ -51,7 +49,6 @@ following command: ----------------------------------- sudo bin/elasticsearch-plugin install analysis-icu ----------------------------------- -// NOTCONSOLE This command will install the version of the plugin that matches your Elasticsearch version and also show a progress bar while downloading. @@ -65,7 +62,6 @@ A plugin can also be downloaded directly from a custom location by specifying th ----------------------------------- sudo bin/elasticsearch-plugin install [url] <1> ----------------------------------- -// NOTCONSOLE <1> must be a valid URL, the plugin name is determined from its descriptor. For instance, to install a plugin from your local file system, you could run: @@ -74,7 +70,6 @@ For instance, to install a plugin from your local file system, you could run: ----------------------------------- sudo bin/elasticsearch-plugin install file:///path/to/plugin.zip ----------------------------------- -// NOTCONSOLE The plugin script will refuse to talk to an HTTPS URL with an untrusted certificate. To use a self-signed HTTPS cert, you will need to add the CA cert @@ -84,7 +79,6 @@ to a local Java truststore and pass the location to the script as follows: ----------------------------------- sudo ES_JAVA_OPTS="-Djavax.net.ssl.trustStore=/path/to/trustStore.jks" bin/elasticsearch-plugin install https://.... ----------------------------------- -// NOTCONSOLE [[listing-removing]] === Listing and Removing Installed Plugins @@ -98,7 +92,6 @@ A list of the currently loaded plugins can be retrieved with the `list` option: ----------------------------------- sudo bin/elasticsearch-plugin list ----------------------------------- -// NOTCONSOLE Alternatively, use the {ref}/cluster-nodes-info.html[node-info API] to find out which plugins are installed on each node in the cluster @@ -113,7 +106,6 @@ Plugins can be removed manually, by deleting the appropriate directory under ----------------------------------- sudo bin/elasticsearch-plugin remove [pluginname] ----------------------------------- -// NOTCONSOLE After a Java plugin has been removed, you will need to restart the node to complete the removal process. @@ -145,7 +137,6 @@ can do this as follows: --------------------- sudo bin/elasticsearch-plugin -Epath.conf=/path/to/custom/config/dir install --------------------- -// NOTCONSOLE You can also set the `CONF_DIR` environment variable to the custom config directory path. @@ -168,7 +159,6 @@ sudo bin/elasticsearch-plugin install analysis-icu --timeout 1m # Wait forever (default) sudo bin/elasticsearch-plugin install analysis-icu --timeout 0 ----------------------------------- -// NOTCONSOLE [float] === Proxy settings @@ -181,7 +171,6 @@ and `http.proxyPort` (or `https.proxyHost` and `https.proxyPort`): ----------------------------------- sudo ES_JAVA_OPTS="-Dhttp.proxyHost=host_name -Dhttp.proxyPort=port_number -Dhttps.proxyHost=host_name -Dhttps.proxyPort=https_port_number" bin/elasticsearch-plugin install analysis-icu ----------------------------------- -// NOTCONSOLE Or on Windows: @@ -190,7 +179,6 @@ Or on Windows: set ES_JAVA_OPTS="-DproxyHost=host_name -DproxyPort=port_number -Dhttps.proxyHost=host_name -Dhttps.proxyPort=https_port_number" bin/elasticsearch-plugin install analysis-icu ------------------------------------ -// NOTCONSOLE === Plugins directory diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 726f55cc889..03466f0c643 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -14,7 +14,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install repository-azure ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -29,7 +28,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove repository-azure ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc index a9658e1f219..d0a5c748439 100644 --- a/docs/plugins/repository-gcs.asciidoc +++ b/docs/plugins/repository-gcs.asciidoc @@ -14,7 +14,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install repository-gcs ---------------------------------------------------------------- -// NOTCONSOLE NOTE: The plugin requires new permission to be installed in order to work @@ -31,7 +30,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove repository-gcs ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. @@ -133,6 +131,7 @@ A service account file looks like this: "client_x509_cert_url": "..." } ---- +// NOTCONSOLE This file must be copied in the `config` directory of the elasticsearch installation and on every node of the cluster. diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc index 02239a78b15..62b1d2a95ca 100644 --- a/docs/plugins/repository-hdfs.asciidoc +++ b/docs/plugins/repository-hdfs.asciidoc @@ -14,7 +14,6 @@ This plugin can be installed through the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install repository-hdfs ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on _every_ node in the cluster, and each node must be restarted after installation. @@ -29,7 +28,6 @@ The plugin can be removed by specifying the _installed_ package: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove repository-hdfs ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index cf9d3248287..5848b827c63 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -16,7 +16,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install repository-s3 ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -31,7 +30,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove repository-s3 ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. @@ -303,6 +301,7 @@ IAM in conjunction with pre-existing S3 buckets. Here is an example policy which "Version": "2012-10-17" } ---- +// NOTCONSOLE You may further restrict the permissions by specifying a prefix within the bucket, in this example, named "foo". @@ -346,6 +345,7 @@ You may further restrict the permissions by specifying a prefix within the bucke "Version": "2012-10-17" } ---- +// NOTCONSOLE The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. If you want elasticsearch to create the bucket instead, you can add the permission to create a @@ -363,6 +363,7 @@ specific bucket like this: ] } ---- +// NOTCONSOLE [[repository-s3-endpoint]] ===== Using other S3 endpoint diff --git a/docs/plugins/store-smb.asciidoc b/docs/plugins/store-smb.asciidoc index ac35342f2f8..731894ae0a8 100644 --- a/docs/plugins/store-smb.asciidoc +++ b/docs/plugins/store-smb.asciidoc @@ -13,7 +13,6 @@ This plugin can be installed using the plugin manager: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin install store-smb ---------------------------------------------------------------- -// NOTCONSOLE The plugin must be installed on every node in the cluster, and each node must be restarted after installation. @@ -28,7 +27,6 @@ The plugin can be removed with the following command: ---------------------------------------------------------------- sudo bin/elasticsearch-plugin remove store-smb ---------------------------------------------------------------- -// NOTCONSOLE The node must be stopped before removing the plugin. diff --git a/docs/python/index.asciidoc b/docs/python/index.asciidoc index d81a20ea392..8440ed1cd0d 100644 --- a/docs/python/index.asciidoc +++ b/docs/python/index.asciidoc @@ -35,7 +35,6 @@ It can be installed with pip: ------------------------------------ pip install elasticsearch ------------------------------------ -// NOTCONSOLE === Versioning @@ -49,6 +48,7 @@ later, 0.4 releases are meant to work with Elasticsearch 0.90.*. The recommended way to set your requirements in your `setup.py` or `requirements.txt` is: +[source,txt] ------------------------------------ # Elasticsearch 2.x elasticsearch>=2.0.0,<3.0.0 diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index b54a288b587..0336e21c2ea 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -227,12 +227,14 @@ a multi-value metrics aggregation, and in case of a single-value metrics aggrega The path must be defined in the following form: +// https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form +[source,ebnf] -------------------------------------------------- -AGG_SEPARATOR := '>' -METRIC_SEPARATOR := '.' -AGG_NAME := -METRIC := -PATH := []*[] +AGG_SEPARATOR = '>' ; +METRIC_SEPARATOR = '.' ; +AGG_NAME = ; +METRIC = ; +PATH = [ , ]* [ , ] ; -------------------------------------------------- [source,js] diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 2f130be11a2..68b2e8511f9 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -344,12 +344,14 @@ a multi-value metrics aggregation, and in case of a single-value metrics aggrega The path must be defined in the following form: +// https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form +[source,ebnf] -------------------------------------------------- -AGG_SEPARATOR := '>' -METRIC_SEPARATOR := '.' -AGG_NAME := -METRIC := -PATH := []*[] +AGG_SEPARATOR = '>' ; +METRIC_SEPARATOR = '.' ; +AGG_NAME = ; +METRIC = ; +PATH = [ , ]* [ , ] ; -------------------------------------------------- [source,js] diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index 73d7f3c26bb..77fc7dfcd5a 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -71,6 +71,7 @@ The following chart shows how the error varies before and after the threshold: //// To generate this chart use this gnuplot script: +[source,gnuplot] ------- #!/usr/bin/gnuplot reset @@ -95,6 +96,7 @@ plot "test.dat" using 1:2 title "threshold=100", \ and generate data in a 'test.dat' file using the below Java code: +[source,java] ------- private static double error(HyperLogLogPlusPlus h, long expected) { double actual = h.cardinality(0); @@ -140,7 +142,7 @@ counting millions of items. On string fields that have a high cardinality, it might be faster to store the hash of your field values in your index and then run the cardinality aggregation on this field. This can either be done by providing hash values from client-side -or by letting elasticsearch compute hash values for you by using the +or by letting elasticsearch compute hash values for you by using the {plugins}/mapper-murmur3.html[`mapper-murmur3`] plugin. NOTE: Pre-computing hashes is usually only useful on very large and/or diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index 499438256e6..c9cd14d6203 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -34,12 +34,14 @@ will be included in the final output. Most pipeline aggregations require another aggregation as their input. The input aggregation is defined via the `buckets_path` parameter, which follows a specific format: +// https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form +[source,ebnf] -------------------------------------------------- -AGG_SEPARATOR := '>' -METRIC_SEPARATOR := '.' -AGG_NAME := -METRIC := -PATH := []*[] +AGG_SEPARATOR = '>' ; +METRIC_SEPARATOR = '.' ; +AGG_NAME = ; +METRIC = ; +PATH = [ , ]* [ , ] ; -------------------------------------------------- For example, the path `"my_bucket>my_stats.avg"` will path to the `avg` value in the `"my_stats"` metric, which is diff --git a/docs/reference/analysis/tokenfilters.asciidoc b/docs/reference/analysis/tokenfilters.asciidoc index 15a911b8ed9..89cce11a615 100644 --- a/docs/reference/analysis/tokenfilters.asciidoc +++ b/docs/reference/analysis/tokenfilters.asciidoc @@ -87,4 +87,6 @@ include::tokenfilters/apostrophe-tokenfilter.asciidoc[] include::tokenfilters/decimal-digit-tokenfilter.asciidoc[] -include::tokenfilters/fingerprint-tokenfilter.asciidoc[] \ No newline at end of file +include::tokenfilters/fingerprint-tokenfilter.asciidoc[] + +include::tokenfilters/minhash-tokenfilter.asciidoc[] \ No newline at end of file diff --git a/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc new file mode 100644 index 00000000000..0660b0b0e90 --- /dev/null +++ b/docs/reference/analysis/tokenfilters/minhash-tokenfilter.asciidoc @@ -0,0 +1,22 @@ +[[analysis-minhash-tokenfilter]] +== Minhash Token Filter + +A token filter of type `min_hash` hashes each token of the token stream and divides +the resulting hashes into buckets, keeping the lowest-valued hashes per +bucket. It then returns these hashes as tokens. + +The following are settings that can be set for a `min_hash` token filter. + +[cols="<,<", options="header",] +|======================================================================= +|Setting |Description +|`hash_count` |The number of hashes to hash the token stream with. Defaults to `1`. + +|`bucket_count` |The number of buckets to divide the minhashes into. Defaults to `512`. + +|`hash_set_size` |The number of minhashes to keep per bucket. Defaults to `1`. + +|`with_rotation` |Whether or not to fill empty buckets with the value of the first non-empty +bucket to its circular right. Only takes effect if hash_set_size is equal to one. +Defaults to `true` if bucket_count is greater than one, else `false`. +|======================================================================= diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index ec223722c68..769a8a4e10d 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -5,6 +5,8 @@ The cluster allocation explanation API is designed to assist in answering the question "why is this shard unassigned?". To explain the allocation (on unassigned state) of a shard, issue a request like: +experimental[The cluster allocation explain API is new and should still be considered experimental. The API may change in ways that are not backwards compatible] + [source,js] -------------------------------------------------- $ curl -XGET 'http://localhost:9200/_cluster/allocation/explain' -d'{ diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 4bd4abb37f2..dc73b4408e6 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -13,6 +13,8 @@ GET _cluster/health // TEST[s/^/PUT test1\n/] Returns this: + +[source,js] -------------------------------------------------- { "cluster_name" : "testcluster", diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 036bd59ad64..427ef0a84dc 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -1,15 +1,16 @@ [[elasticsearch-reference]] = Elasticsearch Reference -:version: 5.0.0-alpha5 -:major-version: 5.x -:branch: master -:jdk: 1.8.0_73 -:defguide: https://www.elastic.co/guide/en/elasticsearch/guide/master -:plugins: https://www.elastic.co/guide/en/elasticsearch/plugins/master -:javaclient: https://www.elastic.co/guide/en/elasticsearch/client/java-api/master/ -:issue: https://github.com/elastic/elasticsearch/issues/ -:pull: https://github.com/elastic/elasticsearch/pull/ +:version: 5.0.0-alpha5 +:major-version: 5.x +:lucene_version: 6.2.0 +:branch: master +:jdk: 1.8.0_73 +:defguide: https://www.elastic.co/guide/en/elasticsearch/guide/master +:plugins: https://www.elastic.co/guide/en/elasticsearch/plugins/master +:javaclient: https://www.elastic.co/guide/en/elasticsearch/client/java-api/master/ +:issue: https://github.com/elastic/elasticsearch/issues/ +:pull: https://github.com/elastic/elasticsearch/pull/ include::getting-started.asciidoc[] @@ -52,7 +53,3 @@ include::glossary.asciidoc[] include::release-notes.asciidoc[] include::redirects.asciidoc[] - - - - diff --git a/docs/reference/mapping/params/geohash-prefix.asciidoc b/docs/reference/mapping/params/geohash-prefix.asciidoc index 33bd21bdeb6..51dfc829947 100644 --- a/docs/reference/mapping/params/geohash-prefix.asciidoc +++ b/docs/reference/mapping/params/geohash-prefix.asciidoc @@ -41,6 +41,7 @@ PUT my_index } } -------------------------------------------------- +// CONSOLE // TEST[warning:geo_point geohash_precision parameter is deprecated and will be removed in the next major release] // TEST[warning:geo_point geohash_prefix parameter is deprecated and will be removed in the next major release] // TEST[warning:geo_point geohash parameter is deprecated and will be removed in the next major release] diff --git a/docs/reference/mapping/params/lat-lon.asciidoc b/docs/reference/mapping/params/lat-lon.asciidoc index 002f91ef3f3..234c652c932 100644 --- a/docs/reference/mapping/params/lat-lon.asciidoc +++ b/docs/reference/mapping/params/lat-lon.asciidoc @@ -31,6 +31,7 @@ PUT my_index } } -------------------------------------------------- +// CONSOLE // TEST[warning:geo_point lat_lon parameter is deprecated and will be removed in the next major release] <1> Setting `lat_lon` to true indexes the geo-point in the `location.lat` and `location.lon` fields. diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc index f173a0df958..fa51e72c0ff 100644 --- a/docs/reference/migration/migrate_5_0/percolator.asciidoc +++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc @@ -24,6 +24,10 @@ Instead a <> must be configured prior to index Indices with a `.percolator` type created on a version before 5.0.0 can still be used, but new indices no longer accept the `.percolator` type. +However it is strongly recommended to reindex any indices containing percolator queries created prior +upgrading to Elasticsearch 5. By doing this the `percolate` query utilize the extracted terms the `percolator` +field type extracted from the percolator queries and potentially execute many times faster. + ==== Percolate document mapping The `percolate` query no longer modifies the mappings. Before the percolate API @@ -53,6 +57,22 @@ The percolate stats have been removed. This is because the percolator no longer The percolator no longer accepts percolator queries containing `range` queries with ranges that are based on current time (using `now`). +==== Percolator queries containing scripts. + +Percolator queries that contain scripts (For example: `script` query or a `function_score` query script function) that +have no explicit language specified will use the Painless scripting language from version 5.0 and up. + +Scripts with no explicit language set in percolator queries stored in indices created prior to version 5.0 +will use the language that has been configured in the `script.legacy.default_lang` setting. This setting defaults to +the Groovy scripting language, which was the default for versions prior to 5.0. If your default scripting language was +different then set the `script.legacy.default_lang` setting to the language you used before. + +In order to make use of the new `percolator` field type all percolator queries should be reindexed into a new index. +When reindexing percolator queries with scripts that have no explicit language defined into a new index, one of the +following two things should be done in order to make the scripts work: +* (Recommended approach) While reindexing the percolator documents, migrate the scripts to the Painless scripting language. +* or add `lang` parameter on the script and set it the language these scripts were written in. + ==== Java client The percolator is no longer part of the core elasticsearch dependency. It has moved to the percolator module. diff --git a/docs/reference/migration/migrate_5_0/scripting.asciidoc b/docs/reference/migration/migrate_5_0/scripting.asciidoc index 4d42ae98b46..3e0db9e1cbd 100644 --- a/docs/reference/migration/migrate_5_0/scripting.asciidoc +++ b/docs/reference/migration/migrate_5_0/scripting.asciidoc @@ -9,8 +9,6 @@ to help make the transition between languages as simple as possible. Documentation for Painless can be found at <> -It is also possible to set the default language back to Groovy using the following setting: `script.default_lang: groovy` - One common difference to note between Groovy and Painless is the use of parameters -- all parameters in Painless must be prefixed with `params.` now. The following example shows the difference: @@ -48,6 +46,12 @@ Painless (`my_modifer` is prefixed with `params`): } ----------------------------------- +The `script.default_lang` setting has been removed. It is no longer possible set the default scripting language. If a +different language than `painless` is used then this should be explicitly specified on the script itself. + +For scripts with no explicit language defined, that are part of already stored percolator queries, the default language +can be controlled with the `script.legacy.default_lang` setting. + ==== Removed 1.x script and template syntax The deprecated 1.x syntax of defining inline scripts / templates and referring to file or index base scripts / templates diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc index fec167c7b84..293e8bb573c 100644 --- a/docs/reference/search/suggesters/context-suggest.asciidoc +++ b/docs/reference/search/suggesters/context-suggest.asciidoc @@ -71,6 +71,7 @@ PUT place_path_category } } -------------------------------------------------- +// CONSOLE // TESTSETUP <1> Defines a `category` context named 'place_type' where the categories must be sent with the suggestions. @@ -330,6 +331,7 @@ POST place/_suggest?pretty } } -------------------------------------------------- +// CONSOLE // TEST[continued] <1> The context query filters for suggestions that fall under the geo location represented by a geohash of '(43.662, -79.380)' diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc index 755650e824d..dace399d650 100644 --- a/docs/reference/search/suggesters/phrase-suggest.asciidoc +++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc @@ -77,6 +77,7 @@ POST test/test {"title": "nobel prize"} POST _refresh -------------------------------------------------- +// CONSOLE // TESTSETUP Once you have the analyzers and mappings set up you can use the `phrase` diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 8a2730c0148..5fb4ad9b7ce 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -111,6 +111,7 @@ GET twitter/tweet/_validate/query?q=post_date:foo&explain=true responds with: +[source,js] -------------------------------------------------- { "valid" : false, diff --git a/docs/reference/setup/install/check-running.asciidoc b/docs/reference/setup/install/check-running.asciidoc index db4783c3dfa..2e255ec35e5 100644 --- a/docs/reference/setup/install/check-running.asciidoc +++ b/docs/reference/setup/install/check-running.asciidoc @@ -3,26 +3,32 @@ You can test that your Elasticsearch node is running by sending an HTTP request to port `9200` on `localhost`: -[source,sh] +[source,js] -------------------------------------------- -curl localhost:9200 +GET / -------------------------------------------- +// CONSOLE which should give you a response something like this: -[source,js] +["source","js",subs="attributes,callouts"] -------------------------------------------- { - "name" : "Harry Leland", + "name" : "Cp8oag6", "cluster_name" : "elasticsearch", "version" : { - "number" : "5.0.0-alpha1", + "number" : "{version}", "build_hash" : "f27399d", "build_date" : "2016-03-30T09:51:41.449Z", "build_snapshot" : false, - "lucene_version" : "6.0.0" + "lucene_version" : "{lucene_version}" }, "tagline" : "You Know, for Search" } -------------------------------------------- - +// TESTRESPONSE[s/"name" : "Cp8oag6",/"name" : "$body.name",/] +// TESTRESPONSE[s/"cluster_name" : "elasticsearch",/"cluster_name" : "$body.cluster_name",/] +// TESTRESPONSE[s/"build_hash" : "f27399d",/"build_hash" : "$body.version.build_hash",/] +// TESTRESPONSE[s/"build_date" : "2016-03-30T09:51:41.449Z",/"build_date" : $body.version.build_date,/] +// TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/] +// So much s/// but at least we test that the layout is close to matching.... diff --git a/docs/reference/setup/sysconfig/heap_size.asciidoc b/docs/reference/setup/sysconfig/heap_size.asciidoc index 6bb32097a0a..00c4553b97f 100644 --- a/docs/reference/setup/sysconfig/heap_size.asciidoc +++ b/docs/reference/setup/sysconfig/heap_size.asciidoc @@ -63,7 +63,6 @@ in the jvm.options file and setting these values via `ES_JAVA_OPTS`: ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch <1> ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2> ------------------ -// NOTCONSOLE <1> Set the minimum and maximum heap size to 2 GB. <2> Set the minimum and maximum heap size to 4000 MB. diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc index b3a6e0aa806..ce95e52fbeb 100644 --- a/docs/reference/setup/sysconfig/swap.asciidoc +++ b/docs/reference/setup/sysconfig/swap.asciidoc @@ -70,7 +70,6 @@ specifying a new temp directory, by starting Elasticsearch with: -------------- ./bin/elasticsearch -Djava.io.tmpdir=/path/to/temp/dir -------------- -// NOTCONSOLE or using the `ES_JAVA_OPTS` environment variable: @@ -79,7 +78,6 @@ or using the `ES_JAVA_OPTS` environment variable: export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir" ./bin/elasticsearch -------------- -// NOTCONSOLE [[disable-swap-files]] ==== Disable all swap files diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc index fe7daf8cca4..0bf99b2fafa 100644 --- a/docs/reference/testing/testing-framework.asciidoc +++ b/docs/reference/testing/testing-framework.asciidoc @@ -20,7 +20,7 @@ All of the tests are run using a custom junit runner, the `RandomizedRunner` pro First, you need to include the testing dependency in your project, along with the elasticsearch dependency you have already added. If you use maven and its `pom.xml` file, it looks like this -[[source,xml]] +[source,xml] -------------------------------------------------- @@ -258,5 +258,3 @@ assertHitCount(searchResponse, 4); assertFirstHit(searchResponse, hasId("4")); assertSearchHits(searchResponse, "1", "2", "3", "4"); ---------------------------- - - diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index 011f949c860..8fed78aca32 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -24,7 +24,7 @@ esplugin { } dependencies { - compile "com.github.spullara.mustache.java:compiler:0.9.1" + compile "com.github.spullara.mustache.java:compiler:0.9.3" } integTest { diff --git a/modules/lang-mustache/licenses/compiler-0.9.1.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.1.jar.sha1 deleted file mode 100644 index 96152e075b3..00000000000 --- a/modules/lang-mustache/licenses/compiler-0.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -14aec5344639782ee76441401b773946c65eb2b3 diff --git a/modules/lang-mustache/licenses/compiler-0.9.3.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.3.jar.sha1 new file mode 100644 index 00000000000..2b0fbbc542e --- /dev/null +++ b/modules/lang-mustache/licenses/compiler-0.9.3.jar.sha1 @@ -0,0 +1 @@ +2815e016c63bec4f18704ea4f5489106a5b01a99 \ No newline at end of file diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 3d5965b3586..09e8afbca0b 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -515,7 +515,8 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder> nodePlugins() { - return Collections.singleton(PercolatorPlugin.class); + return Arrays.asList(PercolatorPlugin.class, FoolMeScriptLang.class); } @Override @@ -81,25 +88,43 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase { .setTypes(".percolator") .addSort("_uid", SortOrder.ASC) .get(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(4L)); assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(3).id(), equalTo("4")); + assertThat(XContentMapValues.extractValue("query.script.script.inline", + searchResponse.getHits().getAt(3).sourceAsMap()), equalTo("return true")); + // we don't upgrade the script definitions so that they include explicitly the lang, + // because we read / parse the query at search time. + assertThat(XContentMapValues.extractValue("query.script.script.lang", + searchResponse.getHits().getAt(3).sourceAsMap()), nullValue()); // verify percolate response PercolateResponse percolateResponse = preparePercolate(client()) + .setIndices(INDEX_NAME) + .setDocumentType("message") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .get(); + + assertThat(percolateResponse.getCount(), equalTo(1L)); + assertThat(percolateResponse.getMatches().length, equalTo(1)); + assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("4")); + + percolateResponse = preparePercolate(client()) .setIndices(INDEX_NAME) .setDocumentType("message") .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog")) .get(); - assertThat(percolateResponse.getCount(), equalTo(2L)); - assertThat(percolateResponse.getMatches().length, equalTo(2)); + assertThat(percolateResponse.getCount(), equalTo(3L)); + assertThat(percolateResponse.getMatches().length, equalTo(3)); assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1")); assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2")); + assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4")); // add an extra query and verify the results - client().prepareIndex(INDEX_NAME, ".percolator", "4") + client().prepareIndex(INDEX_NAME, ".percolator", "5") .setSource(jsonBuilder().startObject().field("query", matchQuery("message", "fox jumps")).endObject()) .get(); refresh(); @@ -110,8 +135,8 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase { .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog")) .get(); - assertThat(percolateResponse.getCount(), equalTo(3L)); - assertThat(percolateResponse.getMatches().length, equalTo(3)); + assertThat(percolateResponse.getCount(), equalTo(4L)); + assertThat(percolateResponse.getMatches().length, equalTo(4)); assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1")); assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2")); assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4")); @@ -131,4 +156,19 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase { ensureGreen(INDEX_NAME); } + // Fool the script service that this is the groovy script language, + // so that we can run a script that has no lang defined implicetely against the legacy language: + public static class FoolMeScriptLang extends MockScriptPlugin { + + @Override + protected Map, Object>> pluginScripts() { + return Collections.singletonMap("return true", (vars) -> true); + } + + @Override + public String pluginScriptLang() { + return "groovy"; + } + } + } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index df1e6ea6f8c..621cb07d3cd 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -36,10 +36,14 @@ import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; @@ -61,6 +65,8 @@ import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; @@ -72,6 +78,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Function; import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -100,7 +108,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, PercolatorPlugin.class); + return pluginList(InternalSettingsPlugin.class, PercolatorPlugin.class, FoolMeScriptPlugin.class); } @Before @@ -493,4 +501,71 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase { DocumentMapper defaultMapper = parser2x.parse("type1", new CompressedXContent(mapping)); assertEquals(mapping, defaultMapper.mappingSource().string()); } + + public void testImplicitlySetDefaultScriptLang() throws Exception { + addQueryMapping(); + XContentBuilder query = jsonBuilder(); + query.startObject(); + query.startObject("script"); + if (randomBoolean()) { + query.field("script", "return true"); + } else { + query.startObject("script"); + query.field("inline", "return true"); + query.endObject(); + } + query.endObject(); + query.endObject(); + + ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", + XContentFactory.jsonBuilder().startObject() + .rawField(fieldName, new BytesArray(query.string())) + .endObject().bytes()); + BytesRef querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); + Map parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); + assertEquals(Script.DEFAULT_SCRIPT_LANG, XContentMapValues.extractValue("script.script.lang", parsedQuery)); + + query = jsonBuilder(); + query.startObject(); + query.startObject("function_score"); + query.startArray("functions"); + query.startObject(); + query.startObject("script_score"); + if (randomBoolean()) { + query.field("script", "return true"); + } else { + query.startObject("script"); + query.field("inline", "return true"); + query.endObject(); + } + query.endObject(); + query.endObject(); + query.endArray(); + query.endObject(); + query.endObject(); + + doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", + XContentFactory.jsonBuilder().startObject() + .rawField(fieldName, new BytesArray(query.string())) + .endObject().bytes()); + querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); + parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2(); + assertEquals(Script.DEFAULT_SCRIPT_LANG, + ((List) XContentMapValues.extractValue("function_score.functions.script_score.script.lang", parsedQuery)).get(0)); + } + + // Just so that we store scripts in percolator queries, but not really execute these scripts. + public static class FoolMeScriptPlugin extends MockScriptPlugin { + + @Override + protected Map, Object>> pluginScripts() { + return Collections.singletonMap("return true", (vars) -> true); + } + + @Override + public String pluginScriptLang() { + return Script.DEFAULT_SCRIPT_LANG; + } + } + } diff --git a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip index f0e2d05e4af..43a8cceb193 100644 Binary files a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip and b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip differ diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 5c49e0f16ea..9aa04185e35 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -37,16 +37,30 @@ import javax.naming.directory.SearchResult; /** * Evaluate Precision at N, N being the number of search results to consider for precision calculation. - * * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. + * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the precision + * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ public class PrecisionAtN extends RankedListQualityMetric { /** Number of results to check against a given set of relevant results. */ private int n; + /** ratings equal or above this value will be considered relevant. */ + private int relevantRatingThreshhold = 1; + public static final String NAME = "precisionatn"; + private static final ParseField SIZE_FIELD = new ParseField("size"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "precision_at", a -> new PrecisionAtN((Integer) a[0])); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + PARSER.declareInt(PrecisionAtN::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + } + public PrecisionAtN(StreamInput in) throws IOException { n = in.readInt(); } @@ -82,12 +96,19 @@ public class PrecisionAtN extends RankedListQualityMetric { return n; } - private static final ParseField SIZE_FIELD = new ParseField("size"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "precision_at", a -> new PrecisionAtN((Integer) a[0])); + /** + * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. + * */ + public void setRelevantRatingThreshhold(int threshold) { + this.relevantRatingThreshhold = threshold; + } - static { - PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + /** + * Return the rating threshold above which ratings are considered to be "relevant" for this metric. + * Defaults to 1. + * */ + public int getRelevantRatingThreshold() { + return relevantRatingThreshhold ; } public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { @@ -103,9 +124,9 @@ public class PrecisionAtN extends RankedListQualityMetric { Collection relevantDocIds = new ArrayList<>(); Collection irrelevantDocIds = new ArrayList<>(); for (RatedDocument doc : ratedDocs) { - if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + if (doc.getRating() >= this.relevantRatingThreshhold) { relevantDocIds.add(doc.getKey()); - } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + } else { irrelevantDocIds.add(doc.getKey()); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index dd4e710859b..3279c40734c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.index.rankeval.PrecisionAtN.RatingMapping; import org.elasticsearch.search.SearchHit; import java.io.IOException; @@ -41,6 +39,8 @@ import javax.naming.directory.SearchResult; /** * Evaluate reciprocal rank. + * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal rank + * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ public class ReciprocalRank extends RankedListQualityMetric { @@ -48,6 +48,9 @@ public class ReciprocalRank extends RankedListQualityMetric { public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK; + /** ratings equal or above this value will be considered relevant. */ + private int relevantRatingThreshhold = 1; + /** * Initializes maxAcceptableRank with 10 */ @@ -90,6 +93,21 @@ public class ReciprocalRank extends RankedListQualityMetric { return this.maxAcceptableRank; } + /** + * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. + * */ + public void setRelevantRatingThreshhold(int threshold) { + this.relevantRatingThreshhold = threshold; + } + + /** + * Return the rating threshold above which ratings are considered to be "relevant" for this metric. + * Defaults to 1. + * */ + public int getRelevantRatingThreshold() { + return relevantRatingThreshhold ; + } + /** * Compute ReciprocalRank based on provided relevant document IDs. * @return reciprocal Rank for above {@link SearchResult} list. @@ -99,9 +117,9 @@ public class ReciprocalRank extends RankedListQualityMetric { Set relevantDocIds = new HashSet<>(); Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { - if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + if (doc.getRating() >= this.relevantRatingThreshhold) { relevantDocIds.add(doc.getKey()); - } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + } else { irrelevantDocIds.add(doc.getKey()); } } @@ -110,16 +128,14 @@ public class ReciprocalRank extends RankedListQualityMetric { int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { - // TODO here we use index/type/id triple not for a rated document but an unrated document in the search hits. Maybe rename? - RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - if (relevantDocIds.contains(id)) { + RatedDocumentKey key = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); + if (relevantDocIds.contains(key)) { if (found == false && i < maxAcceptableRank) { - firstRelevant = i + 1; // add one because rank is not - // 0-based + firstRelevant = i + 1; // add one because rank is not 0-based found = true; } } else { - unknownDocIds.add(id); + unknownDocIds.add(key); } } @@ -133,11 +149,13 @@ public class ReciprocalRank extends RankedListQualityMetric { } private static final ParseField MAX_RANK_FIELD = new ParseField("max_acceptable_rank"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); static { PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD); + PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index d668b21630e..381ddc10023 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -64,6 +64,27 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } + /** + * test that the relevant rating threshold can be set to something larger than 1. + * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant + */ + public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + PrecisionAtN precisionAtN = new PrecisionAtN(5); + precisionAtN.setRelevantRatingThreshhold(2); + assertEquals((double) 3 / 5, precisionAtN.evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "0"), Rating.RELEVANT.ordinal())); @@ -96,11 +117,13 @@ public class PrecisionAtNTests extends ESTestCase { public void testParseFromXContent() throws IOException { String xContent = " {\n" - + " \"size\": 10\n" + + " \"size\": 10,\n" + + " \"relevant_rating_threshold\" : 2" + "}"; XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); assertEquals(10, precicionAt.getN()); + assertEquals(2, precicionAt.getRelevantRatingThreshold()); } public void testCombine() { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b524e763dc7..44c90d1eca3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -26,10 +26,12 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Vector; +import java.util.concurrent.ExecutionException; import static java.util.Collections.emptyList; @@ -103,6 +105,29 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); } + /** + * test that the relevant rating threshold can be set to something larger than 1. + * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first relevant doc has + * third ranking position, so RR should be 1/3 + */ + public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + + ReciprocalRank reciprocalRank = new ReciprocalRank(); + reciprocalRank.setRelevantRatingThreshhold(2); + assertEquals((double) 1 / 3, reciprocalRank.evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testCombine() { ReciprocalRank reciprocalRank = new ReciprocalRank(); Vector partialResults = new Vector<>(3); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index e2910be64f0..ab4f00492b0 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -19,6 +19,14 @@ package org.elasticsearch.plugins; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -26,13 +34,8 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.cli.UserException; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.test.ESTestCase; -import org.junit.Before; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; @LuceneTestCase.SuppressFileSystems("*") public class RemovePluginCommandTests extends ESTestCase { @@ -109,4 +112,26 @@ public class RemovePluginCommandTests extends ESTestCase { assertRemoveCleaned(env); } + public void testConfigDirPreserved() throws Exception { + Files.createDirectories(env.pluginsFile().resolve("fake")); + final Path configDir = env.configFile().resolve("fake"); + Files.createDirectories(configDir); + Files.createFile(configDir.resolve("fake.yml")); + final MockTerminal terminal = removePlugin("fake", home); + assertTrue(Files.exists(env.configFile().resolve("fake"))); + assertThat(terminal.getOutput(), containsString(expectedConfigDirPreservedMessage(configDir))); + assertRemoveCleaned(env); + } + + public void testNoConfigDirPreserved() throws Exception { + Files.createDirectories(env.pluginsFile().resolve("fake")); + final Path configDir = env.configFile().resolve("fake"); + final MockTerminal terminal = removePlugin("fake", home); + assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir)))); + } + + private String expectedConfigDirPreservedMessage(final Path configDir) { + return "-> Preserving plugin config files [" + configDir + "] in case of upgrade, delete manually if not needed"; + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 1b4fcd2922a..c752563b804 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory; import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory; import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory; import org.elasticsearch.index.analysis.MappingCharFilterFactory; +import org.elasticsearch.index.analysis.MinHashTokenFilterFactory; import org.elasticsearch.index.analysis.MultiTermAwareComponent; import org.elasticsearch.index.analysis.NGramTokenFilterFactory; import org.elasticsearch.index.analysis.NGramTokenizerFactory; @@ -93,7 +94,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -/** +/** * Alerts us if new analyzers are added to lucene, so we don't miss them. *

* If we don't want to expose one for a specific reason, just map it to Void. @@ -115,11 +116,11 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("thai", ThaiTokenizerFactory.class) .put("uax29urlemail", UAX29URLEmailTokenizerFactory.class) .put("whitespace", WhitespaceTokenizerFactory.class) - + // this one "seems to mess up offsets". probably shouldn't be a tokenizer... .put("wikipedia", Void.class) .immutableMap(); - + static final Map> KNOWN_TOKENFILTERS = new MapBuilder>() // exposed in ES .put("apostrophe", ApostropheFilterFactory.class) @@ -184,6 +185,7 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("scandinaviannormalization", ScandinavianNormalizationFilterFactory.class) .put("serbiannormalization", SerbianNormalizationFilterFactory.class) .put("shingle", ShingleTokenFilterFactory.class) + .put("minhash", MinHashTokenFilterFactory.class) .put("snowballporter", SnowballTokenFilterFactory.class) .put("soraninormalization", SoraniNormalizationFilterFactory.class) .put("soranistem", StemmerTokenFilterFactory.class) @@ -199,7 +201,7 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("type", KeepTypesFilterFactory.class) .put("uppercase", UpperCaseTokenFilterFactory.class) .put("worddelimiter", WordDelimiterTokenFilterFactory.class) - + // TODO: these tokenfilters are not yet exposed: useful? // suggest stop @@ -228,16 +230,15 @@ public class AnalysisFactoryTestCase extends ESTestCase { .put("fingerprint", Void.class) // for tee-sinks .put("daterecognizer", Void.class) - .put("minhash", Void.class) .immutableMap(); - + static final Map> KNOWN_CHARFILTERS = new MapBuilder>() // exposed in ES .put("htmlstrip", HtmlStripCharFilterFactory.class) .put("mapping", MappingCharFilterFactory.class) .put("patternreplace", PatternReplaceCharFilterFactory.class) - + // TODO: these charfilters are not yet exposed: useful? // handling of zwnj for persian .put("persian", Void.class) diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java index 5b1c1adfb80..3c5f105e4d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java +++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java @@ -27,14 +27,17 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.junit.Assert; import java.io.IOException; import java.util.Random; +import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; @@ -54,11 +57,11 @@ public class BackgroundIndexer implements AutoCloseable { final CopyOnWriteArrayList failures; final AtomicBoolean stop = new AtomicBoolean(false); final AtomicLong idGenerator = new AtomicLong(); - final AtomicLong indexCounter = new AtomicLong(); final CountDownLatch startLatch = new CountDownLatch(1); final AtomicBoolean hasBudget = new AtomicBoolean(false); // when set to true, writers will acquire writes from a semaphore final Semaphore availableBudget = new Semaphore(0); final boolean useAutoGeneratedIDs; + private final Set ids = ConcurrentCollections.newConcurrentSet(); volatile int minFieldSize = 10; volatile int maxFieldSize = 140; @@ -158,7 +161,8 @@ public class BackgroundIndexer implements AutoCloseable { BulkResponse bulkResponse = bulkRequest.get(); for (BulkItemResponse bulkItemResponse : bulkResponse) { if (!bulkItemResponse.isFailed()) { - indexCounter.incrementAndGet(); + boolean add = ids.add(bulkItemResponse.getId()); + assert add : "ID: " + bulkItemResponse.getId() + " already used"; } else { throw new ElasticsearchException("bulk request failure, id: [" + bulkItemResponse.getFailure().getId() + "] message: " + bulkItemResponse.getFailure().getMessage()); @@ -173,14 +177,17 @@ public class BackgroundIndexer implements AutoCloseable { } id = idGenerator.incrementAndGet(); if (useAutoGeneratedIDs) { - client.prepareIndex(index, type).setSource(generateSource(id, threadRandom)).get(); + IndexResponse indexResponse = client.prepareIndex(index, type).setSource(generateSource(id, threadRandom)).get(); + boolean add = ids.add(indexResponse.getId()); + assert add : "ID: " + indexResponse.getId() + " already used"; } else { - client.prepareIndex(index, type, Long.toString(id)).setSource(generateSource(id, threadRandom)).get(); + IndexResponse indexResponse = client.prepareIndex(index, type, Long.toString(id)).setSource(generateSource(id, threadRandom)).get(); + boolean add = ids.add(indexResponse.getId()); + assert add : "ID: " + indexResponse.getId() + " already used"; } - indexCounter.incrementAndGet(); } } - logger.info("**** done indexing thread {} stop: {} numDocsIndexed: {}", indexerId, stop.get(), indexCounter.get()); + logger.info("**** done indexing thread {} stop: {} numDocsIndexed: {}", indexerId, stop.get(), ids.size()); } catch (Exception e) { failures.add(e); final long docId = id; @@ -274,7 +281,7 @@ public class BackgroundIndexer implements AutoCloseable { } public long totalIndexedDocs() { - return indexCounter.get(); + return ids.size(); } public Throwable[] getFailures() { @@ -299,4 +306,11 @@ public class BackgroundIndexer implements AutoCloseable { public void close() throws Exception { stop(); } + + /** + * Returns the ID set of all documents indexed by this indexer run + */ + public Set getIds() { + return this.ids; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 0e411a6efc8..adc3e1ec2d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -134,6 +134,7 @@ public abstract class ESTestCase extends LuceneTestCase { // this will fail System.setProperty("es.log4j.shutdownEnabled", "false"); System.setProperty("log4j2.disable.jmx", "true"); + System.setProperty("log4j.skipJansi", "true"); // jython has this crazy shaded Jansi version that log4j2 tries to load BootstrapForTesting.ensureInitialized(); }