honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/rank-eval

This commit is contained in:
Christoph Büscher 2017-05-30 14:24:26 +02:00
parent 10d308578e efcbfb7c32
commit 3d6fb4eb0b
560 changed files with 17170 additions and 7682 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
build.gradle
View File

@ -62,7 +62,12 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
}
}
// introspect all versions of ES that may be tested agains for backwards compatibility
/* Introspect all versions of ES that may be tested agains for backwards
* compatibility. It is *super* important that this logic is the same as the
* logic in VersionUtils.java, modulo alphas, betas, and rcs which are ignored
* in gradle because they don't have any backwards compatibility guarantees
* but are not ignored in VersionUtils.java because the tests expect them not
* to be. */
Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
int prevMajor = currentVersion.major - 1
File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
@ -72,13 +77,14 @@ List<Version> versions = []
int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
for (String line : versionLines) {
Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/
/* Note that this skips alphas and betas which is fine because they aren't
* compatible with anything. */
Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+) .*/
if (match.matches()) {
int major = Integer.parseInt(match.group(1))
int minor = Integer.parseInt(match.group(2))
int bugfix = Integer.parseInt(match.group(3))
boolean unreleased = match.group(4) != null
Version foundVersion = new Version(major, minor, bugfix, false, unreleased)
Version foundVersion = new Version(major, minor, bugfix, false)
if (currentVersion != foundVersion) {
versions.add(foundVersion)
}
@ -96,10 +102,13 @@ if (currentVersion.bugfix == 0) {
// If on a release branch, after the initial release of that branch, the bugfix version will
// be bumped, and will be != 0. On master and N.x branches, we want to test against the
// unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
// and the bwc-zip distribution will checkout and build that version.
// and the bwc distribution will checkout and build that version.
Version last = versions[-1]
versions[-1] = new Version(last.major, last.minor, last.bugfix,
true, last.unreleased)
versions[-1] = new Version(last.major, last.minor, last.bugfix, true)
if (last.bugfix == 0) {
versions[-2] = new Version(
versions[-2].major, versions[-2].minor, versions[-2].bugfix, true)
}
}
// injecting groovy property variables into all projects
@ -114,6 +123,44 @@ allprojects {
}
}
task('verifyVersions') {
description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
group 'Verification'
enabled = false == gradle.startParameter.isOffline()
doLast {
// Read the list from maven central
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })
// Limit the known versions to those that should be index compatible
knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }
/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<String> actualVersions = new TreeSet<>(
indexCompatVersions
.findAll { false == it.snapshot }
.collect { it.toString() })
// TODO this is almost certainly going to fail on 5.4 when we release 5.5.0
// Finally, compare!
if (!knownVersions.equals(actualVersions)) {
throw new GradleException("out-of-date versions\nActual :" +
actualVersions + "\nExpected:" + knownVersions +
"; update Version.java")
}
}
}
task('precommit') {
dependsOn(verifyVersions)
}
subprojects {
project.afterEvaluate {
// include license and notice in jars
@ -168,12 +215,25 @@ subprojects {
"org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
"org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
"org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
"org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
]
if (wireCompatVersions[-1].snapshot) {
// if the most previous version is a snapshot, we need to connect that version to the
// bwc-zip project which will checkout and build that snapshot version
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip'
if (indexCompatVersions[-1].snapshot) {
/* The last and second to last versions can be snapshots. Rather than use
* snapshots built by CI we connect these versions to projects that build
* those those versions from the HEAD of the appropriate branch. */
if (indexCompatVersions[-1].bugfix == 0) {
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
} else {
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
}
}
project.afterEvaluate {
configurations.all {

50
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -311,7 +311,13 @@ class BuildPlugin implements Plugin<Project> {
/**
* Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
*
* The current fixup is to set compile time deps back to compile from runtime (known issue with maven-publish plugin).
* <ul>
* <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
* as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
* being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
* translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
* <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
* </ul>
*/
private static Closure fixupDependencies(Project project) {
return { XmlProvider xml ->
@ -321,15 +327,53 @@ class BuildPlugin implements Plugin<Project> {
return
}
// fix deps incorrectly marked as runtime back to compile time deps
// see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
// check each dependency for any transitive deps
for (Node depNode : depsNodes.get(0).children()) {
String groupId = depNode.get('groupId').get(0).text()
String artifactId = depNode.get('artifactId').get(0).text()
String version = depNode.get('version').get(0).text()
// fix deps incorrectly marked as runtime back to compile time deps
// see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
boolean isCompileDep = project.configurations.compile.allDependencies.find { dep ->
dep.name == depNode.artifactId.text()
}
if (depNode.scope.text() == 'runtime' && isCompileDep) {
depNode.scope*.value = 'compile'
}
// remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards
// see https://github.com/elastic/elasticsearch/issues/24490
NodeList exclusionsNode = depNode.get('exclusions')
if (exclusionsNode.size() > 0) {
depNode.remove(exclusionsNode.get(0))
}
// collect the transitive deps now that we know what this dependency is
String depConfig = transitiveDepConfigName(groupId, artifactId, version)
Configuration configuration = project.configurations.findByName(depConfig)
if (configuration == null) {
continue // we did not make this dep non-transitive
}
Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
if (artifacts.size() <= 1) {
// this dep has no transitive deps (or the only artifact is itself)
continue
}
// we now know we have something to exclude, so add exclusions for all artifacts except the main one
Node exclusions = depNode.appendNode('exclusions')
for (ResolvedArtifact artifact : artifacts) {
ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
String depGroupId = moduleVersionIdentifier.group
String depArtifactId = moduleVersionIdentifier.name
// add exclusions for all artifacts except the main one
if (depGroupId != groupId || depArtifactId != artifactId) {
Node exclusion = exclusions.appendNode('exclusion')
exclusion.appendNode('groupId', depGroupId)
exclusion.appendNode('artifactId', depArtifactId)
}
}
}
}
}

10
buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy
View File

@ -29,20 +29,14 @@ public class Version {
final int bugfix
final int id
final boolean snapshot
/**
* Is the vesion listed as {@code _UNRELEASED} in Version.java.
*/
final boolean unreleased
public Version(int major, int minor, int bugfix, boolean snapshot,
boolean unreleased) {
public Version(int major, int minor, int bugfix, boolean snapshot) {
this.major = major
this.minor = minor
this.bugfix = bugfix
this.snapshot = snapshot
this.id = major * 100000 + minor * 1000 + bugfix * 10 +
(snapshot ? 1 : 0)
this.unreleased = unreleased
}
public static Version fromString(String s) {
@ -54,7 +48,7 @@ public class Version {
bugfix = bugfix.split('-')[0]
}
return new Version(parts[0] as int, parts[1] as int, bugfix as int,
snapshot, false)
snapshot)
}
@Override

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
View File

@ -168,6 +168,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
current.println(" - skip:")
current.println(" features: ")
current.println(" - stash_in_key")
current.println(" - stash_in_path")
current.println(" - stash_path_replace")
current.println(" - warnings")
}
if (test.skipTest) {

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
View File

@ -90,6 +90,7 @@ public class SnippetsTask extends DefaultTask {
* tests cleaner.
*/
subst = subst.replace('$body', '\\$body')
subst = subst.replace('$_path', '\\$_path')
// \n is a new line....
subst = subst.replace('\\n', '\n')
snippet.contents = snippet.contents.replaceAll(

6
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View File

@ -46,11 +46,11 @@ class ClusterConfiguration {
int transportPort = 0
/**
* An override of the data directory. This may only be used with a single node.
* The value is lazily evaluated at runtime as a String path.
* An override of the data directory. Input is the node number and output
* is the override data directory.
*/
@Input
Object dataDir = null
Closure<String> dataDir = null
/** Optional override of the cluster name. */
@Input

22
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -169,7 +169,7 @@ class ClusterFormationTasks {
if (node.config.plugins.isEmpty() == false) {
if (node.nodeVersion == VersionProperties.elasticsearch) {
setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node)
setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node, prefix)
} else {
setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node, prefix)
}
@ -184,7 +184,7 @@ class ClusterFormationTasks {
// install plugins
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue())
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
}
// sets up any extra config files that need to be copied over to the ES instance;
@ -379,7 +379,7 @@ class ClusterFormationTasks {
* For each plugin, if the plugin has rest spec apis in its tests, those api files are also copied
* to the test resources for this project.
*/
static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node) {
static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)
List<FileCollection> pluginFiles = []
@ -387,7 +387,7 @@ class ClusterFormationTasks {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = "_plugin_${pluginProject.path}"
String configurationName = "_plugin_${prefix}_${pluginProject.path}"
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
@ -422,7 +422,7 @@ class ClusterFormationTasks {
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = "_plugin_bwc_${pluginProject.path}"
String configurationName = "_plugin_bwc_${prefix}_${pluginProject.path}"
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
@ -457,12 +457,12 @@ class ClusterFormationTasks {
return installModule
}
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin) {
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
final FileCollection pluginZip;
if (node.nodeVersion != VersionProperties.elasticsearch) {
pluginZip = project.configurations.getByName("_plugin_bwc_${plugin.path}")
pluginZip = project.configurations.getByName("_plugin_bwc_${prefix}_${plugin.path}")
} else {
pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
}
// delay reading the file location until execution time by wrapping in a closure within a GString
Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
@ -657,11 +657,11 @@ class ClusterFormationTasks {
standardOutput = new ByteArrayOutputStream()
doLast {
String out = standardOutput.toString()
if (out.contains("${pid} org.elasticsearch.bootstrap.Elasticsearch") == false) {
if (out.contains("${ext.pid} org.elasticsearch.bootstrap.Elasticsearch") == false) {
logger.error('jps -l')
logger.error(out)
logger.error("pid file: ${pidFile}")
logger.error("pid: ${pid}")
logger.error("pid file: ${node.pidFile}")
logger.error("pid: ${ext.pid}")
throw new GradleException("jps -l did not report any process with org.elasticsearch.bootstrap.Elasticsearch\n" +
"Did you run gradle clean? Maybe an old pid file is still lying around.")
} else {

5
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
View File

@ -111,10 +111,7 @@ class NodeInfo {
homeDir = homeDir(baseDir, config.distribution, nodeVersion)
confDir = confDir(baseDir, config.distribution, nodeVersion)
if (config.dataDir != null) {
if (config.numNodes != 1) {
throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node")
}
dataDir = config.dataDir
dataDir = "${config.dataDir(nodeNum)}"
} else {
dataDir = new File(homeDir, "data")
}

6
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -1,7 +1,7 @@
<?xml version="1.0"?>
<!DOCTYPE suppressions PUBLIC
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
<suppressions>
<!-- On Windows, Checkstyle matches files using \ path separator -->
@ -795,4 +795,4 @@
<suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
<suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
<suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
</suppressions>
</suppressions>

2
buildSrc/version.properties
View File

@ -1,6 +1,6 @@
# When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
elasticsearch = 6.0.0-alpha2
lucene = 7.0.0-snapshot-89f6d17
lucene = 7.0.0-snapshot-a0aef2f
# optional dependencies
spatial4j = 0.6

4
client/rest-high-level/build.gradle
View File

@ -26,11 +26,11 @@ group = 'org.elasticsearch.client'
dependencies {
compile "org.elasticsearch:elasticsearch:${version}"
compile "org.elasticsearch.client:rest:${version}"
compile "org.elasticsearch.plugin:parent-join-client:${version}"
compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
testCompile "org.elasticsearch.client:test:${version}"
testCompile "org.elasticsearch.test:framework:${version}"
// for parent/child testing
testCompile "org.elasticsearch.plugin:parent-join-client:${version}"
testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
testCompile "junit:junit:${versions.junit}"
testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"

70
client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
View File

@ -33,7 +33,9 @@ import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.Nullable;
@ -42,11 +44,13 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import java.io.ByteArrayOutputStream;
@ -59,7 +63,7 @@ import java.util.StringJoiner;
final class Request {
private static final String DELIMITER = "/";
private static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
final String method;
final String endpoint;
@ -79,6 +83,7 @@ final class Request {
"method='" + method + '\'' +
", endpoint='" + endpoint + '\'' +
", params=" + params +
", hasBody=" + (entity != null) +
'}';
}
@ -307,23 +312,48 @@ final class Request {
xContentType = Requests.INDEX_CONTENT_TYPE;
}
BytesRef source = XContentHelper.toXContent(updateRequest, xContentType, false).toBytesRef();
HttpEntity entity = new ByteArrayEntity(source.bytes, source.offset, source.length, ContentType.create(xContentType.mediaType()));
HttpEntity entity = createEntity(updateRequest, xContentType);
return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), entity);
}
static Request search(SearchRequest searchRequest) throws IOException {
String endpoint = endpoint(searchRequest.indices(), searchRequest.types(), "_search");
Params params = Params.builder();
params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
params.withRouting(searchRequest.routing());
params.withPreference(searchRequest.preference());
params.withIndicesOptions(searchRequest.indicesOptions());
params.putParam("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
if (searchRequest.requestCache() != null) {
params.putParam("request_cache", Boolean.toString(searchRequest.requestCache()));
}
params.putParam("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
if (searchRequest.scroll() != null) {
params.putParam("scroll", searchRequest.scroll().keepAlive());
}
HttpEntity entity = null;
if (searchRequest.source() != null) {
entity = createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE);
}
return new Request(HttpGet.METHOD_NAME, endpoint, params.getParams(), entity);
}
private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
return new ByteArrayEntity(source.bytes, source.offset, source.length, ContentType.create(xContentType.mediaType()));
}
static String endpoint(String[] indices, String[] types, String endpoint) {
return endpoint(String.join(",", indices), String.join(",", types), endpoint);
}
/**
* Utility method to build request's endpoint.
*/
static String endpoint(String... parts) {
if (parts == null || parts.length == 0) {
return DELIMITER;
}
StringJoiner joiner = new StringJoiner(DELIMITER, DELIMITER, "");
StringJoiner joiner = new StringJoiner("/", "/", "");
for (String part : parts) {
if (part != null) {
if (Strings.hasLength(part)) {
joiner.add(part);
}
}
@ -453,6 +483,26 @@ final class Request {
return this;
}
Params withIndicesOptions (IndicesOptions indicesOptions) {
putParam("ignore_unavailable", Boolean.toString(indicesOptions.ignoreUnavailable()));
putParam("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
String expandWildcards;
if (indicesOptions.expandWildcardsOpen() == false && indicesOptions.expandWildcardsClosed() == false) {
expandWildcards = "none";
} else {
StringJoiner joiner = new StringJoiner(",");
if (indicesOptions.expandWildcardsOpen()) {
joiner.add("open");
}
if (indicesOptions.expandWildcardsClosed()) {
joiner.add("closed");
}
expandWildcards = joiner.toString();
}
putParam("expand_wildcards", expandWildcards);
return this;
}
Map<String, String> getParams() {
return Collections.unmodifiableMap(params);
}

180
client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
View File

@ -36,22 +36,117 @@ import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.main.MainRequest;
import org.elasticsearch.action.main.MainResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ContextParser;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
import org.elasticsearch.join.aggregations.ParsedChildren;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.filters.ParsedFilters;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested;
import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.range.date.ParsedDateRange;
import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.range.geodistance.ParsedGeoDistance;
import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms;
import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms;
import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
import org.elasticsearch.search.aggregations.matrix.stats.ParsedMatrixStats;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality;
import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.geobounds.ParsedGeoBounds;
import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.geocentroid.ParsedGeoCentroid;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.max.ParsedMax;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.ParsedMin;
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentileRanks;
import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.ParsedHDRPercentiles;
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentileRanks;
import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.ParsedTDigestPercentiles;
import org.elasticsearch.search.aggregations.metrics.scripted.ParsedScriptedMetric;
import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ParsedExtendedStats;
import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.ParsedBucketMetricValue;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.ParsedPercentilesBucket;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.ParsedStatsBucket;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ParsedExtendedStatsBucket;
import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
import org.elasticsearch.search.suggest.term.TermSuggestion;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.util.Collections.emptySet;
@ -82,10 +177,8 @@ public class RestHighLevelClient {
*/
protected RestHighLevelClient(RestClient restClient, List<NamedXContentRegistry.Entry> namedXContentEntries) {
this.client = Objects.requireNonNull(restClient);
this.registry = new NamedXContentRegistry(Stream.of(
getNamedXContents().stream(),
namedXContentEntries.stream()
).flatMap(Function.identity()).collect(toList()));
this.registry = new NamedXContentRegistry(Stream.of(getDefaultNamedXContents().stream(), namedXContentEntries.stream())
.flatMap(Function.identity()).collect(toList()));
}
/**
@ -214,6 +307,24 @@ public class RestHighLevelClient {
Collections.singleton(404), headers);
}
/**
* Executes a search using the Search api
*
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on elastic.co</a>
*/
public SearchResponse search(SearchRequest searchRequest, Header... headers) throws IOException {
return performRequestAndParseEntity(searchRequest, Request::search, SearchResponse::fromXContent, emptySet(), headers);
}
/**
* Asynchronously executes a search using the Search api
*
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on elastic.co</a>
*/
public void searchAsync(SearchRequest searchRequest, ActionListener<SearchResponse> listener, Header... headers) {
performRequestAsyncAndParseEntity(searchRequest, Request::search, SearchResponse::fromXContent, listener, emptySet(), headers);
}
private <Req extends ActionRequest, Resp> Resp performRequestAndParseEntity(Req request,
CheckedFunction<Req, Request, IOException> requestConverter,
CheckedFunction<XContentParser, Resp, IOException> entityParser,
@ -365,9 +476,60 @@ public class RestHighLevelClient {
return response.getStatusLine().getStatusCode() == 200;
}
static List<NamedXContentRegistry.Entry> getNamedXContents() {
List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
//namedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField("sterms"), StringTerms::fromXContent));
return namedXContents;
static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
map.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c));
map.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c));
map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
map.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c));
map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c));
map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c));
map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c));
map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c));
map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c));
map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME,
(p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
map.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c));
map.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c));
map.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c));
map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
map.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));
map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c));
map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c));
map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
map.put(ChildrenAggregationBuilder.NAME, (p, c) -> ParsedChildren.fromXContent(p, (String) c));
map.put(MatrixStatsAggregationBuilder.NAME, (p, c) -> ParsedMatrixStats.fromXContent(p, (String) c));
List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
.map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
.collect(Collectors.toList());
entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestion.NAME),
(parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestion.NAME),
(parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestion.NAME),
(parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
return entries;
}
}

21
client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
View File

@ -55,7 +55,6 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static java.util.Collections.singletonMap;
@ -63,16 +62,6 @@ import static java.util.Collections.singletonMap;
public class CrudIT extends ESRestHighLevelClientTestCase {
public void testDelete() throws IOException {
{
// Testing non existing document
String docId = "does_not_exist";
DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId);
DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
assertEquals("index", deleteResponse.getIndex());
assertEquals("type", deleteResponse.getType());
assertEquals(docId, deleteResponse.getId());
assertEquals(DocWriteResponse.Result.NOT_FOUND, deleteResponse.getResult());
}
{
// Testing deletion
String docId = "id";
@ -87,6 +76,16 @@ public class CrudIT extends ESRestHighLevelClientTestCase {
assertEquals(docId, deleteResponse.getId());
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
}
{
// Testing non existing document
String docId = "does_not_exist";
DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId);
DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
assertEquals("index", deleteResponse.getIndex());
assertEquals("type", deleteResponse.getType());
assertEquals(docId, deleteResponse.getId());
assertEquals(DocWriteResponse.Result.NOT_FOUND, deleteResponse.getResult());
}
{
// Testing version conflict
String docId = "version_conflict";

144
client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
View File

@ -21,25 +21,41 @@ package org.elasticsearch.client;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkShardRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.rescore.QueryRescorerBuilder;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.RandomObjects;
@ -48,6 +64,7 @@ import java.io.InputStream;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.StringJoiner;
import java.util.function.Consumer;
import java.util.function.Function;
@ -257,9 +274,8 @@ public class RequestTests extends ESTestCase {
assertEquals(method, request.method);
HttpEntity entity = request.entity;
assertNotNull(entity);
assertTrue(entity instanceof ByteArrayEntity);
assertEquals(indexRequest.getContentType().mediaType(), entity.getContentType().getValue());
try (XContentParser parser = createParser(xContentType.xContent(), entity.getContent())) {
assertEquals(nbFields, parser.map().size());
}
@ -353,7 +369,6 @@ public class RequestTests extends ESTestCase {
assertEquals("POST", request.method);
HttpEntity entity = request.entity;
assertNotNull(entity);
assertTrue(entity instanceof ByteArrayEntity);
UpdateRequest parsedUpdateRequest = new UpdateRequest();
@ -470,7 +485,7 @@ public class RequestTests extends ESTestCase {
assertEquals("/_bulk", request.endpoint);
assertEquals(expectedParams, request.params);
assertEquals("POST", request.method);
assertEquals(xContentType.mediaType(), request.entity.getContentType().getValue());
byte[] content = new byte[(int) request.entity.getContentLength()];
try (InputStream inputStream = request.entity.getContent()) {
Streams.readFully(inputStream, content);
@ -584,6 +599,127 @@ public class RequestTests extends ESTestCase {
}
}
public void testSearch() throws Exception {
SearchRequest searchRequest = new SearchRequest();
int numIndices = randomIntBetween(0, 5);
String[] indices = new String[numIndices];
for (int i = 0; i < numIndices; i++) {
indices[i] = "index-" + randomAlphaOfLengthBetween(2, 5);
}
searchRequest.indices(indices);
int numTypes = randomIntBetween(0, 5);
String[] types = new String[numTypes];
for (int i = 0; i < numTypes; i++) {
types[i] = "type-" + randomAlphaOfLengthBetween(2, 5);
}
searchRequest.types(types);
Map<String, String> expectedParams = new HashMap<>();
expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
if (randomBoolean()) {
searchRequest.routing(randomAlphaOfLengthBetween(3, 10));
expectedParams.put("routing", searchRequest.routing());
}
if (randomBoolean()) {
searchRequest.preference(randomAlphaOfLengthBetween(3, 10));
expectedParams.put("preference", searchRequest.preference());
}
if (randomBoolean()) {
searchRequest.searchType(randomFrom(SearchType.values()));
}
expectedParams.put("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
if (randomBoolean()) {
searchRequest.requestCache(randomBoolean());
expectedParams.put("request_cache", Boolean.toString(searchRequest.requestCache()));
}
if (randomBoolean()) {
searchRequest.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE));
}
expectedParams.put("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
if (randomBoolean()) {
searchRequest.scroll(randomTimeValue());
expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep());
}
if (randomBoolean()) {
searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
}
expectedParams.put("ignore_unavailable", Boolean.toString(searchRequest.indicesOptions().ignoreUnavailable()));
expectedParams.put("allow_no_indices", Boolean.toString(searchRequest.indicesOptions().allowNoIndices()));
if (searchRequest.indicesOptions().expandWildcardsOpen() && searchRequest.indicesOptions().expandWildcardsClosed()) {
expectedParams.put("expand_wildcards", "open,closed");
} else if (searchRequest.indicesOptions().expandWildcardsOpen()) {
expectedParams.put("expand_wildcards", "open");
} else if (searchRequest.indicesOptions().expandWildcardsClosed()) {
expectedParams.put("expand_wildcards", "closed");
} else {
expectedParams.put("expand_wildcards", "none");
}
SearchSourceBuilder searchSourceBuilder = null;
if (frequently()) {
searchSourceBuilder = new SearchSourceBuilder();
if (randomBoolean()) {
searchSourceBuilder.size(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
searchSourceBuilder.from(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
searchSourceBuilder.minScore(randomFloat());
}
if (randomBoolean()) {
searchSourceBuilder.explain(randomBoolean());
}
if (randomBoolean()) {
searchSourceBuilder.profile(randomBoolean());
}
if (randomBoolean()) {
searchSourceBuilder.highlighter(new HighlightBuilder().field(randomAlphaOfLengthBetween(3, 10)));
}
if (randomBoolean()) {
searchSourceBuilder.query(new TermQueryBuilder(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)));
}
if (randomBoolean()) {
searchSourceBuilder.aggregation(new TermsAggregationBuilder(randomAlphaOfLengthBetween(3, 10), ValueType.STRING)
.field(randomAlphaOfLengthBetween(3, 10)));
}
if (randomBoolean()) {
searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion(randomAlphaOfLengthBetween(3, 10),
new CompletionSuggestionBuilder(randomAlphaOfLengthBetween(3, 10))));
}
if (randomBoolean()) {
searchSourceBuilder.addRescorer(new QueryRescorerBuilder(
new TermQueryBuilder(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10))));
}
if (randomBoolean()) {
searchSourceBuilder.collapse(new CollapseBuilder(randomAlphaOfLengthBetween(3, 10)));
}
searchRequest.source(searchSourceBuilder);
}
Request request = Request.search(searchRequest);
StringJoiner endpoint = new StringJoiner("/", "/", "");
String index = String.join(",", indices);
if (Strings.hasLength(index)) {
endpoint.add(index);
}
String type = String.join(",", types);
if (Strings.hasLength(type)) {
endpoint.add(type);
}
endpoint.add("_search");
assertEquals(endpoint.toString(), request.endpoint);
assertEquals(expectedParams, request.params);
if (searchSourceBuilder == null) {
assertNull(request.entity);
} else {
BytesReference expectedBytes = XContentHelper.toXContent(searchSourceBuilder, XContentType.JSON, false);
assertEquals(XContentType.JSON.mediaType(), request.entity.getContentType().getValue());
assertEquals(expectedBytes, new BytesArray(EntityUtils.toByteArray(request.entity)));
}
}
public void testParams() {
final int nbParams = randomIntBetween(0, 10);
Request.Params params = Request.Params.builder();

19
client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.client;
import com.fasterxml.jackson.core.JsonParseException;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
@ -51,6 +50,8 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.cbor.CborXContent;
import org.elasticsearch.common.xcontent.smile.SmileXContent;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import org.mockito.ArgumentMatcher;
@ -61,7 +62,9 @@ import org.mockito.internal.matchers.VarargMatcher;
import java.io.IOException;
import java.net.SocketTimeoutException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@ -566,8 +569,18 @@ public class RestHighLevelClientTests extends ESTestCase {
}
public void testNamedXContents() {
List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getNamedXContents();
assertEquals(0, namedXContents.size());
List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getDefaultNamedXContents();
assertEquals(45, namedXContents.size());
Map<Class<?>, Integer> categories = new HashMap<>();
for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1);
if (counter != null) {
categories.put(namedXContent.categoryClass, counter + 1);
}
}
assertEquals(2, categories.size());
assertEquals(Integer.valueOf(42), categories.get(Aggregation.class));
assertEquals(Integer.valueOf(3), categories.get(Suggest.Suggestion.class));
}
private static class TrackingActionListener implements ActionListener<Integer> {

395
client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java Normal file
View File

@ -0,0 +1,395 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.join.aggregations.Children;
import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.bucket.range.Range;
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.junit.Before;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.lessThan;
public class SearchIT extends ESRestHighLevelClientTestCase {
@Before
public void indexDocuments() throws IOException {
StringEntity doc1 = new StringEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/index/type/1", Collections.emptyMap(), doc1);
StringEntity doc2 = new StringEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/index/type/2", Collections.emptyMap(), doc2);
StringEntity doc3 = new StringEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/index/type/3", Collections.emptyMap(), doc3);
StringEntity doc4 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/index/type/4", Collections.emptyMap(), doc4);
StringEntity doc5 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/index/type/5", Collections.emptyMap(), doc5);
client().performRequest("POST", "/index/_refresh");
}
public void testSearchNoQuery() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getAggregations());
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(5, searchResponse.getHits().totalHits);
assertEquals(5, searchResponse.getHits().getHits().length);
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
assertEquals("index", searchHit.getIndex());
assertEquals("type", searchHit.getType());
assertThat(Integer.valueOf(searchHit.getId()), both(greaterThan(0)).and(lessThan(6)));
assertEquals(1.0f, searchHit.getScore(), 0);
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertTrue(searchHit.getSourceAsMap().containsKey("type"));
assertTrue(searchHit.getSourceAsMap().containsKey("num"));
assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
}
}
public void testSearchMatchQuery() throws IOException {
SearchRequest searchRequest = new SearchRequest();
searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10)));
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getAggregations());
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(1, searchResponse.getHits().totalHits);
assertEquals(1, searchResponse.getHits().getHits().length);
assertThat(searchResponse.getHits().getMaxScore(), greaterThan(0f));
SearchHit searchHit = searchResponse.getHits().getHits()[0];
assertEquals("index", searchHit.getIndex());
assertEquals("type", searchHit.getType());
assertEquals("1", searchHit.getId());
assertThat(searchHit.getScore(), greaterThan(0f));
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertEquals("type1", searchHit.getSourceAsMap().get("type"));
assertEquals(50, searchHit.getSourceAsMap().get("num2"));
}
public void testSearchWithTermsAgg() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword"));
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
Terms termsAgg = searchResponse.getAggregations().get("agg1");
assertEquals("agg1", termsAgg.getName());
assertEquals(2, termsAgg.getBuckets().size());
Terms.Bucket type1 = termsAgg.getBucketByKey("type1");
assertEquals(3, type1.getDocCount());
assertEquals(0, type1.getAggregations().asList().size());
Terms.Bucket type2 = termsAgg.getBucketByKey("type2");
assertEquals(2, type2.getDocCount());
assertEquals(0, type2.getAggregations().asList().size());
}
public void testSearchWithRangeAgg() throws IOException {
{
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num"));
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
() -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync));
assertEquals(RestStatus.BAD_REQUEST, exception.status());
}
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num")
.addRange("first", 0, 30).addRange("second", 31, 200));
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertThat(searchResponse.getTook().nanos(), greaterThan(0L));
assertEquals(5, searchResponse.getHits().totalHits);
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
Range rangeAgg = searchResponse.getAggregations().get("agg1");
assertEquals("agg1", rangeAgg.getName());
assertEquals(2, rangeAgg.getBuckets().size());
{
Range.Bucket bucket = rangeAgg.getBuckets().get(0);
assertEquals("first", bucket.getKeyAsString());
assertEquals(2, bucket.getDocCount());
}
{
Range.Bucket bucket = rangeAgg.getBuckets().get(1);
assertEquals("second", bucket.getKeyAsString());
assertEquals(3, bucket.getDocCount());
}
}
public void testSearchWithTermsAndRangeAgg() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword");
agg.subAggregation(new RangeAggregationBuilder("subagg").field("num")
.addRange("first", 0, 30).addRange("second", 31, 200));
searchSourceBuilder.aggregation(agg);
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
Terms termsAgg = searchResponse.getAggregations().get("agg1");
assertEquals("agg1", termsAgg.getName());
assertEquals(2, termsAgg.getBuckets().size());
Terms.Bucket type1 = termsAgg.getBucketByKey("type1");
assertEquals(3, type1.getDocCount());
assertEquals(1, type1.getAggregations().asList().size());
{
Range rangeAgg = type1.getAggregations().get("subagg");
assertEquals(2, rangeAgg.getBuckets().size());
{
Range.Bucket bucket = rangeAgg.getBuckets().get(0);
assertEquals("first", bucket.getKeyAsString());
assertEquals(2, bucket.getDocCount());
}
{
Range.Bucket bucket = rangeAgg.getBuckets().get(1);
assertEquals("second", bucket.getKeyAsString());
assertEquals(1, bucket.getDocCount());
}
}
Terms.Bucket type2 = termsAgg.getBucketByKey("type2");
assertEquals(2, type2.getDocCount());
assertEquals(1, type2.getAggregations().asList().size());
{
Range rangeAgg = type2.getAggregations().get("subagg");
assertEquals(2, rangeAgg.getBuckets().size());
{
Range.Bucket bucket = rangeAgg.getBuckets().get(0);
assertEquals("first", bucket.getKeyAsString());
assertEquals(0, bucket.getDocCount());
}
{
Range.Bucket bucket = rangeAgg.getBuckets().get(1);
assertEquals("second", bucket.getKeyAsString());
assertEquals(2, bucket.getDocCount());
}
}
}
public void testSearchWithMatrixStats() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(new MatrixStatsAggregationBuilder("agg1").fields(Arrays.asList("num", "num2")));
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertThat(searchResponse.getTook().nanos(), greaterThan(0L));
assertEquals(5, searchResponse.getHits().totalHits);
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
assertEquals(1, searchResponse.getAggregations().asList().size());
MatrixStats matrixStats = searchResponse.getAggregations().get("agg1");
assertEquals(5, matrixStats.getFieldCount("num"));
assertEquals(56d, matrixStats.getMean("num"), 0d);
assertEquals(1830d, matrixStats.getVariance("num"), 0d);
assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d);
assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d);
assertEquals(5, matrixStats.getFieldCount("num2"));
assertEquals(29d, matrixStats.getMean("num2"), 0d);
assertEquals(330d, matrixStats.getVariance("num2"), 0d);
assertEquals(-0.13568039346585542, matrixStats.getSkewness("num2"), 0d);
assertEquals(1.3517561983471074, matrixStats.getKurtosis("num2"), 0d);
assertEquals(-767.5, matrixStats.getCovariance("num", "num2"), 0d);
assertEquals(-0.9876336291667923, matrixStats.getCorrelation("num", "num2"), 0d);
}
public void testSearchWithParentJoin() throws IOException {
StringEntity parentMapping = new StringEntity("{\n" +
" \"mappings\": {\n" +
" \"answer\" : {\n" +
" \"_parent\" : {\n" +
" \"type\" : \"question\"\n" +
" }\n" +
" }\n" +
" },\n" +
" \"settings\": {\n" +
" \"index.mapping.single_type\": false" +
" }\n" +
"}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/child_example", Collections.emptyMap(), parentMapping);
StringEntity questionDoc = new StringEntity("{\n" +
" \"body\": \"<p>I have Windows 2003 server and i bought a new Windows 2008 server...\",\n" +
" \"title\": \"Whats the best way to file transfer my site from server to a newer one?\",\n" +
" \"tags\": [\n" +
" \"windows-server-2003\",\n" +
" \"windows-server-2008\",\n" +
" \"file-transfer\"\n" +
" ]\n" +
"}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/child_example/question/1", Collections.emptyMap(), questionDoc);
StringEntity answerDoc1 = new StringEntity("{\n" +
" \"owner\": {\n" +
" \"location\": \"Norfolk, United Kingdom\",\n" +
" \"display_name\": \"Sam\",\n" +
" \"id\": 48\n" +
" },\n" +
" \"body\": \"<p>Unfortunately you're pretty much limited to FTP...\",\n" +
" \"creation_date\": \"2009-05-04T13:45:37.030\"\n" +
"}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "child_example/answer/1", Collections.singletonMap("parent", "1"), answerDoc1);
StringEntity answerDoc2 = new StringEntity("{\n" +
" \"owner\": {\n" +
" \"location\": \"Norfolk, United Kingdom\",\n" +
" \"display_name\": \"Troll\",\n" +
" \"id\": 49\n" +
" },\n" +
" \"body\": \"<p>Use Linux...\",\n" +
" \"creation_date\": \"2009-05-05T13:45:37.030\"\n" +
"}", ContentType.APPLICATION_JSON);
client().performRequest("PUT", "/child_example/answer/2", Collections.singletonMap("parent", "1"), answerDoc2);
client().performRequest("POST", "/_refresh");
TermsAggregationBuilder leafTermAgg = new TermsAggregationBuilder("top-names", ValueType.STRING)
.field("owner.display_name.keyword").size(10);
ChildrenAggregationBuilder childrenAgg = new ChildrenAggregationBuilder("to-answers", "answer").subAggregation(leafTermAgg);
TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("top-tags", ValueType.STRING).field("tags.keyword")
.size(10).subAggregation(childrenAgg);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0).aggregation(termsAgg);
SearchRequest searchRequest = new SearchRequest("child_example");
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertThat(searchResponse.getTook().nanos(), greaterThan(0L));
assertEquals(3, searchResponse.getHits().totalHits);
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
assertEquals(1, searchResponse.getAggregations().asList().size());
Terms terms = searchResponse.getAggregations().get("top-tags");
assertEquals(0, terms.getDocCountError());
assertEquals(0, terms.getSumOfOtherDocCounts());
assertEquals(3, terms.getBuckets().size());
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket.getKeyAsString(),
either(equalTo("file-transfer")).or(equalTo("windows-server-2003")).or(equalTo("windows-server-2008")));
assertEquals(1, bucket.getDocCount());
assertEquals(1, bucket.getAggregations().asList().size());
Children children = bucket.getAggregations().get("to-answers");
assertEquals(2, children.getDocCount());
assertEquals(1, children.getAggregations().asList().size());
Terms leafTerms = children.getAggregations().get("top-names");
assertEquals(0, leafTerms.getDocCountError());
assertEquals(0, leafTerms.getSumOfOtherDocCounts());
assertEquals(2, leafTerms.getBuckets().size());
assertEquals(2, leafTerms.getBuckets().size());
Terms.Bucket sam = leafTerms.getBucketByKey("Sam");
assertEquals(1, sam.getDocCount());
Terms.Bucket troll = leafTerms.getBucketByKey("Troll");
assertEquals(1, troll.getDocCount());
}
}
public void testSearchWithSuggest() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion("sugg1", new PhraseSuggestionBuilder("type"))
.setGlobalText("type"));
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getAggregations());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(0, searchResponse.getHits().totalHits);
assertEquals(0f, searchResponse.getHits().getMaxScore(), 0f);
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(1, searchResponse.getSuggest().size());
Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> sugg = searchResponse
.getSuggest().iterator().next();
assertEquals("sugg1", sugg.getName());
for (Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option> options : sugg) {
assertEquals("type", options.getText().string());
assertEquals(0, options.getOffset());
assertEquals(4, options.getLength());
assertEquals(2 ,options.getOptions().size());
for (Suggest.Suggestion.Entry.Option option : options) {
assertThat(option.getScore(), greaterThan(0f));
assertThat(option.getText().string(), either(equalTo("type1")).or(equalTo("type2")));
}
}
}
private static void assertSearchHeader(SearchResponse searchResponse) {
assertThat(searchResponse.getTook().nanos(), greaterThan(0L));
assertEquals(0, searchResponse.getFailedShards());
assertThat(searchResponse.getTotalShards(), greaterThan(0));
assertEquals(searchResponse.getTotalShards(), searchResponse.getSuccessfulShards());
assertEquals(0, searchResponse.getShardFailures().length);
}
}

38
client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
View File

@ -55,27 +55,29 @@ public class PreBuiltTransportClient extends TransportClient {
}
/**
* Netty wants to do some unsafe things like use unsafe and replace a private field. This method disables these things by default, but
* can be overridden by setting the corresponding system properties.
* Netty wants to do some unwelcome things like use unsafe and replace a private field, or use a poorly considered buffer recycler. This
* method disables these things by default, but can be overridden by setting the corresponding system properties.
*/
@SuppressForbidden(reason = "set system properties to configure Netty")
private static void initializeNetty() {
final String noUnsafeKey = "io.netty.noUnsafe";
final String noUnsafe = System.getProperty(noUnsafeKey);
if (noUnsafe == null) {
// disable Netty from using unsafe
// while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
// the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
System.setProperty(noUnsafeKey, Boolean.toString(true));
}
/*
* We disable three pieces of Netty functionality here:
* - we disable Netty from being unsafe
* - we disable Netty from replacing the selector key set
* - we disable Netty from using the recycler
*
* While permissions are needed to read and set these, the permissions needed here are innocuous and thus should simply be granted
* rather than us handling a security exception here.
*/
setSystemPropertyIfUnset("io.netty.noUnsafe", Boolean.toString(true));
setSystemPropertyIfUnset("io.netty.noKeySetOptimization", Boolean.toString(true));
setSystemPropertyIfUnset("io.netty.recycler.maxCapacityPerThread", Integer.toString(0));
}
final String noKeySetOptimizationKey = "io.netty.noKeySetOptimization";
final String noKeySetOptimization = System.getProperty(noKeySetOptimizationKey);
if (noKeySetOptimization == null) {
// disable Netty from replacing the selector key set
// while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
// the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
System.setProperty(noKeySetOptimizationKey, Boolean.toString(true));
@SuppressForbidden(reason = "set system properties to configure Netty")
private static void setSystemPropertyIfUnset(final String key, final String value) {
final String currentValue = System.getProperty(key);
if (currentValue == null) {
System.setProperty(key, value);
}
}

40
core/build.gradle
View File

@ -277,43 +277,3 @@ if (isEclipse == false || project.path == ":core-tests") {
check.dependsOn integTest
integTest.mustRunAfter test
}
task('verifyVersions') {
description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
group 'Verification'
enabled = false == gradle.startParameter.isOffline()
doLast {
// Read the list from maven central
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })
// Limit the known versions to those that should be wire compatible
String currentVersion = versions.elasticsearch.minus('-SNAPSHOT')
int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
if (prevMajor == 4) {
// 4 didn't exist, it was 2.
prevMajor = 2;
}
knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }
/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<String> actualVersions = new TreeSet<>(
indexCompatVersions
.findAll { false == it.unreleased }
.collect { it.toString() })
// Finally, compare!
if (!knownVersions.equals(actualVersions)) {
throw new GradleException("out-of-date versions\nActual :" +
actualVersions + "\nExpected:" + knownVersions +
"; update Version.java")
}
}
}
check.dependsOn(verifyVersions)

1
core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e69234c2e898d86a53edbe8d22e33bebc45286cd

1
core/licenses/lucene-analyzers-common-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5e191674c50c9d99c9838da52cbf67c411998f4e

1
core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
48172a8e1fe6562f55ab671d42af53652794d5df

1
core/licenses/lucene-backward-codecs-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
45bc34ab640d5d1a7491b523631b902f20db5384

1
core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
3dab251d4c7ab4ff5095e5f1d1e127ec2cf3c07d

1
core/licenses/lucene-core-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b44d86e9077443c3ba4918a85603734461c6b448

1
core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
c01ae8a23b733d75d058a76bd85fcb49b9fd06fd

1
core/licenses/lucene-grouping-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
409b616d40e2041a02890b2dc477ed845e3121e9

1
core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
c53df048b97946fe66035505306b5651b702adb1

1
core/licenses/lucene-highlighter-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
cfac105541315e2ca54955f681b410a7aa3bbb9d

1
core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
1ecb349ba29abab75359e5125ac8a94fc81441d5

1
core/licenses/lucene-join-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
993c1331130dd26c632b964fd8caac259bb9f3fc

1
core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e5f53b38652b1284ff254fba39e624ec117aef7d

1
core/licenses/lucene-memory-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ec1460a28850410112a6349a7fff27df31242295

1
core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
2f340ed3f46d6b4c89fa31975b675c19028c15eb

1
core/licenses/lucene-misc-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
57d342dbe68cf05361ccfda6bb76f2410cac900b

1
core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
a13862fb62cc1e516d16d6b6bb3cdb906c4925f6

1
core/licenses/lucene-queries-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5ed10847b6a2353ac66decd5a2ee1a1d34353049

1
core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
4e014f72a588453bae7dd1a555d741cf3bf39032

1
core/licenses/lucene-queryparser-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
23ce6c2ea59287d8fe4fe31f466e9a58a1efe7b5

1
core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
5e87d61c604d6b1c0ee5c38f09441d1b8b9c8c2b

1
core/licenses/lucene-sandbox-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
78bda71c8e65428927136f81112a031aa9cd04d4

1
core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
be14aa163b339403d8ec904493c1be5dfa9baeaf

1
core/licenses/lucene-spatial-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
1e7ea95e6197176015b13551c7496be4867ede45

1
core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
a2c13be0fe4c5a98a30ec6ae673be1442409817c

1
core/licenses/lucene-spatial-extras-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5ae4ecd6c478456395ae9a3f954b8afc13629bb9

1
core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
92b8282e474845fdae31f9f239f953bc7164401f

1
core/licenses/lucene-spatial3d-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d5d1a81fc290b9660a49557f848dc2a3c4f2048b

1
core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
1c4aaea267ed41657ebf01769bfddbcab5b27414

1
core/licenses/lucene-suggest-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d77cdd8f2782062a3b4c319c64f0fa4d804aafed

201
core/src/main/java/org/apache/lucene/analysis/miscellaneous/DeDuplicatingTokenFilter.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.analysis.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.hash.MurmurHash3;
import java.io.IOException;
import java.util.ArrayList;
/**
* Inspects token streams for duplicate sequences of tokens. Token sequences
* have a minimum length - 6 is a good heuristic as it avoids filtering common
* idioms/phrases but detects longer sections that are typical of cut+paste
* copies of text.
*
* <p>
* Internally each token is hashed/moduloed into a single byte (so 256 possible
* values for each token) and then recorded in a trie of seen byte sequences
* using a {@link DuplicateByteSequenceSpotter}. This trie is passed into the
* TokenFilter constructor so a single object can be reused across multiple
* documents.
*
* <p>
* The emitDuplicates setting controls if duplicate tokens are filtered from
* results or are output (the {@link DuplicateSequenceAttribute} attribute can
* be used to inspect the number of prior sightings when emitDuplicates is true)
*/
public class DeDuplicatingTokenFilter extends FilteringTokenFilter {
private final DuplicateSequenceAttribute seqAtt = addAttribute(DuplicateSequenceAttribute.class);
private final boolean emitDuplicates;
static final MurmurHash3.Hash128 seed = new MurmurHash3.Hash128();
public DeDuplicatingTokenFilter(TokenStream in, DuplicateByteSequenceSpotter byteStreamDuplicateSpotter) {
this(in, byteStreamDuplicateSpotter, false);
}
/**
*
* @param in
* The input token stream
* @param byteStreamDuplicateSpotter
* object which retains trie of token sequences
* @param emitDuplicates
* true if duplicate tokens are to be emitted (use
* {@link DuplicateSequenceAttribute} attribute to inspect number
* of prior sightings of tokens as part of a sequence).
*/
public DeDuplicatingTokenFilter(TokenStream in, DuplicateByteSequenceSpotter byteStreamDuplicateSpotter, boolean emitDuplicates) {
super(new DuplicateTaggingFilter(byteStreamDuplicateSpotter, in));
this.emitDuplicates = emitDuplicates;
}
@Override
protected boolean accept() throws IOException {
return emitDuplicates || seqAtt.getNumPriorUsesInASequence() < 1;
}
private static class DuplicateTaggingFilter extends TokenFilter {
private final DuplicateSequenceAttribute seqAtt = addAttribute(DuplicateSequenceAttribute.class);
TermToBytesRefAttribute termBytesAtt = addAttribute(TermToBytesRefAttribute.class);
private DuplicateByteSequenceSpotter byteStreamDuplicateSpotter;
private ArrayList<State> allTokens;
int pos = 0;
private final int windowSize;
protected DuplicateTaggingFilter(DuplicateByteSequenceSpotter byteStreamDuplicateSpotter, TokenStream input) {
super(input);
this.byteStreamDuplicateSpotter = byteStreamDuplicateSpotter;
this.windowSize = DuplicateByteSequenceSpotter.TREE_DEPTH;
}
@Override
public final boolean incrementToken() throws IOException {
if (allTokens == null) {
loadAllTokens();
}
clearAttributes();
if (pos < allTokens.size()) {
State earlierToken = allTokens.get(pos);
pos++;
restoreState(earlierToken);
return true;
} else {
return false;
}
}
public void loadAllTokens() throws IOException {
// TODO consider changing this implementation to emit tokens as-we-go
// rather than buffering all. However this array is perhaps not the
// bulk of memory usage (in practice the dupSequenceSpotter requires
// ~5x the original content size in its internal tree ).
allTokens = new ArrayList<State>(256);
/*
* Given the bytes 123456123456 and a duplicate sequence size of 6
* the byteStreamDuplicateSpotter will only flag the final byte as
* part of a duplicate sequence due to the byte-at-a-time streaming
* nature of its assessments. When this happens we retain a buffer
* of the last 6 tokens so that we can mark the states of prior
* tokens (bytes 7 to 11) as also being duplicates
*/
pos = 0;
boolean isWrapped = false;
State priorStatesBuffer[] = new State[windowSize];
short priorMaxNumSightings[] = new short[windowSize];
int cursor = 0;
while (input.incrementToken()) {
BytesRef bytesRef = termBytesAtt.getBytesRef();
long tokenHash = MurmurHash3.hash128(bytesRef.bytes, bytesRef.offset, bytesRef.length, 0, seed).h1;
byte tokenByte = (byte) (tokenHash & 0xFF);
short numSightings = byteStreamDuplicateSpotter.addByte(tokenByte);
priorStatesBuffer[cursor] = captureState();
// Revise prior captured State objects if the latest
// token is marked as a duplicate
if (numSightings >= 1) {
int numLengthsToRecord = windowSize;
int pos = cursor;
while (numLengthsToRecord > 0) {
if (pos < 0) {
pos = windowSize - 1;
}
priorMaxNumSightings[pos] = (short) Math.max(priorMaxNumSightings[pos], numSightings);
numLengthsToRecord--;
pos--;
}
}
// Reposition cursor to next free slot
cursor++;
if (cursor >= windowSize) {
// wrap around the buffer
cursor = 0;
isWrapped = true;
}
// clean out the end of the tail that we may overwrite if the
// next iteration adds a new head
if (isWrapped) {
// tokenPos is now positioned on tail - emit any valid
// tokens we may about to overwrite in the next iteration
if (priorStatesBuffer[cursor] != null) {
recordLengthInfoState(priorMaxNumSightings, priorStatesBuffer, cursor);
}
}
} // end loop reading all tokens from stream
// Flush the buffered tokens
int pos = isWrapped ? nextAfter(cursor) : 0;
while (pos != cursor) {
recordLengthInfoState(priorMaxNumSightings, priorStatesBuffer, pos);
pos = nextAfter(pos);
}
}
private int nextAfter(int pos) {
pos++;
if (pos >= windowSize) {
pos = 0;
}
return pos;
}
private void recordLengthInfoState(short[] maxNumSightings, State[] tokenStates, int cursor) {
if (maxNumSightings[cursor] > 0) {
// We need to patch in the max sequence length we recorded at
// this position into the token state
restoreState(tokenStates[cursor]);
seqAtt.setNumPriorUsesInASequence(maxNumSightings[cursor]);
maxNumSightings[cursor] = 0;
// record the patched state
tokenStates[cursor] = captureState();
}
allTokens.add(tokenStates[cursor]);
}
}
}

311
core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateByteSequenceSpotter.java Normal file
View File

@ -0,0 +1,311 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.util.RamUsageEstimator;
/**
* A Trie structure for analysing byte streams for duplicate sequences. Bytes
* from a stream are added one at a time using the addByte method and the number
* of times it has been seen as part of a sequence is returned.
*
* The minimum required length for a duplicate sequence detected is 6 bytes.
*
* The design goals are to maximize speed of lookup while minimizing the space
* required to do so. This has led to a hybrid solution for representing the
* bytes that make up a sequence in the trie.
*
* If we have 6 bytes in sequence e.g. abcdef then they are represented as
* object nodes in the tree as follows:
* <p>
* (a)-(b)-(c)-(def as an int)
* <p>
*
*
* {@link RootTreeNode} objects are used for the first two levels of the tree
* (representing bytes a and b in the example sequence). The combinations of
* objects at these 2 levels are few so internally these objects allocate an
* array of 256 child node objects to quickly address children by indexing
* directly into the densely packed array using a byte value. The third level in
* the tree holds {@link LightweightTreeNode} nodes that have few children
* (typically much less than 256) and so use a dynamically-grown array to hold
* child nodes as simple int primitives. These ints represent the final 3 bytes
* of a sequence and also hold a count of the number of times the entire sequence
* path has been visited (count is a single byte).
* <p>
* The Trie grows indefinitely as more content is added and while theoretically
* it could be massive (a 6-depth tree could produce 256^6 nodes) non-random
* content e.g English text contains fewer variations.
* <p>
* In future we may look at using one of these strategies when memory is tight:
* <ol>
* <li>auto-pruning methods to remove less-visited parts of the tree
* <li>auto-reset to wipe the whole tree and restart when a memory threshold is
* reached
* <li>halting any growth of the tree
* </ol>
*
* Tests on real-world-text show that the size of the tree is a multiple of the
* input text where that multiplier varies between 10 and 5 times as the content
* size increased from 10 to 100 megabytes of content.
*
*/
public class DuplicateByteSequenceSpotter {
public static final int TREE_DEPTH = 6;
// The maximum number of repetitions that are counted
public static final int MAX_HIT_COUNT = 255;
private final TreeNode root;
private boolean sequenceBufferFilled = false;
private final byte[] sequenceBuffer = new byte[TREE_DEPTH];
private int nextFreePos = 0;
// ==Performance info
private final int[] nodesAllocatedByDepth;
private int nodesResizedByDepth;
// ==== RAM usage estimation settings ====
private long bytesAllocated;
// Root node object plus inner-class reference to containing "this"
// (profiler suggested this was a cost)
static final long TREE_NODE_OBJECT_SIZE = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
// A TreeNode specialization with an array ref (dynamically allocated and
// fixed-size)
static final long ROOT_TREE_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
// A KeyedTreeNode specialization with an array ref (dynamically allocated
// and grown)
static final long LIGHTWEIGHT_TREE_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
// A KeyedTreeNode specialization with a short-based hit count and a
// sequence of bytes encoded as an int
static final long LEAF_NODE_OBJECT_SIZE = TREE_NODE_OBJECT_SIZE + Short.BYTES + Integer.BYTES;
public DuplicateByteSequenceSpotter() {
this.nodesAllocatedByDepth = new int[4];
this.bytesAllocated = 0;
root = new RootTreeNode((byte) 1, null, 0);
}
/**
* Reset the sequence detection logic to avoid any continuation of the
* immediately previous bytes. A minimum of dupSequenceSize bytes need to be
* added before any new duplicate sequences will be reported.
* Hit counts are not reset by calling this method.
*/
public void startNewSequence() {
sequenceBufferFilled = false;
nextFreePos = 0;
}
/**
* Add a byte to the sequence.
* @param b
* the next byte in a sequence
* @return number of times this byte and the preceding 6 bytes have been
* seen before as a sequence (only counts up to 255)
*
*/
public short addByte(byte b) {
// Add latest byte to circular buffer
sequenceBuffer[nextFreePos] = b;
nextFreePos++;
if (nextFreePos >= sequenceBuffer.length) {
nextFreePos = 0;
sequenceBufferFilled = true;
}
if (sequenceBufferFilled == false) {
return 0;
}
TreeNode node = root;
// replay updated sequence of bytes represented in the circular
// buffer starting from the tail
int p = nextFreePos;
// The first tier of nodes are addressed using individual bytes from the
// sequence
node = node.add(sequenceBuffer[p], 0);
p = nextBufferPos(p);
node = node.add(sequenceBuffer[p], 1);
p = nextBufferPos(p);
node = node.add(sequenceBuffer[p], 2);
// The final 3 bytes in the sequence are represented in an int
// where the 4th byte will contain a hit count.
p = nextBufferPos(p);
int sequence = 0xFF & sequenceBuffer[p];
p = nextBufferPos(p);
sequence = sequence << 8 | (0xFF & sequenceBuffer[p]);
p = nextBufferPos(p);
sequence = sequence << 8 | (0xFF & sequenceBuffer[p]);
return (short) (node.add(sequence << 8) - 1);
}
private int nextBufferPos(int p) {
p++;
if (p >= sequenceBuffer.length) {
p = 0;
}
return p;
}
/**
* Base class for nodes in the tree. Subclasses are optimised for use at
* different locations in the tree - speed-optimized nodes represent
* branches near the root while space-optimized nodes are used for deeper
* leaves/branches.
*/
abstract class TreeNode {
TreeNode(byte key, TreeNode parentNode, int depth) {
nodesAllocatedByDepth[depth]++;
}
public abstract TreeNode add(byte b, int depth);
/**
*
* @param byteSequence
* a sequence of bytes encoded as an int
* @return the number of times the full sequence has been seen (counting
* up to a maximum of 32767).
*/
public abstract short add(int byteSequence);
}
// Node implementation for use at the root of the tree that sacrifices space
// for speed.
class RootTreeNode extends TreeNode {
// A null-or-256 sized array that can be indexed into using a byte for
// fast access.
// Being near the root of the tree it is expected that this is a
// non-sparse array.
TreeNode[] children;
RootTreeNode(byte key, TreeNode parentNode, int depth) {
super(key, parentNode, depth);
bytesAllocated += ROOT_TREE_NODE_OBJECT_SIZE;
}
public TreeNode add(byte b, int depth) {
if (children == null) {
children = new TreeNode[256];
bytesAllocated += (RamUsageEstimator.NUM_BYTES_OBJECT_REF * 256);
}
int bIndex = 0xFF & b;
TreeNode node = children[bIndex];
if (node == null) {
if (depth <= 1) {
// Depths 0 and 1 use RootTreeNode impl and create
// RootTreeNodeImpl children
node = new RootTreeNode(b, this, depth);
} else {
// Deeper-level nodes are less visited but more numerous
// so use a more space-friendly data structure
node = new LightweightTreeNode(b, this, depth);
}
children[bIndex] = node;
}
return node;
}
@Override
public short add(int byteSequence) {
throw new UnsupportedOperationException("Root nodes do not support byte sequences encoded as integers");
}
}
// Node implementation for use by the depth 3 branches of the tree that
// sacrifices speed for space.
final class LightweightTreeNode extends TreeNode {
// An array dynamically resized but frequently only sized 1 as most
// sequences leading to end leaves are one-off paths.
// It is scanned for matches sequentially and benchmarks showed
// that sorting contents on insertion didn't improve performance.
int[] children = null;
LightweightTreeNode(byte key, TreeNode parentNode, int depth) {
super(key, parentNode, depth);
bytesAllocated += LIGHTWEIGHT_TREE_NODE_OBJECT_SIZE;
}
@Override
public short add(int byteSequence) {
if (children == null) {
// Create array adding new child with the byte sequence combined with hitcount of 1.
// Most nodes at this level we expect to have only 1 child so we start with the
// smallest possible child array.
children = new int[1];
bytesAllocated += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + Integer.BYTES;
children[0] = byteSequence + 1;
return 1;
}
// Find existing child and if discovered increment count
for (int i = 0; i < children.length; i++) {
int child = children[i];
if (byteSequence == (child & 0xFFFFFF00)) {
int hitCount = child & 0xFF;
if (hitCount < MAX_HIT_COUNT) {
children[i]++;
}
return (short) (hitCount + 1);
}
}
// Grow array adding new child
int[] newChildren = new int[children.length + 1];
bytesAllocated += Integer.BYTES;
System.arraycopy(children, 0, newChildren, 0, children.length);
children = newChildren;
// Combine the byte sequence with a hit count of 1 into an int.
children[newChildren.length - 1] = byteSequence + 1;
nodesResizedByDepth++;
return 1;
}
@Override
public TreeNode add(byte b, int depth) {
throw new UnsupportedOperationException("Leaf nodes do not take byte sequences");
}
}
public final long getEstimatedSizeInBytes() {
return bytesAllocated;
}
/**
* @return Performance info - the number of nodes allocated at each depth
*/
public int[] getNodesAllocatedByDepth() {
return nodesAllocatedByDepth.clone();
}
/**
* @return Performance info - the number of resizing of children arrays, at
* each depth
*/
public int getNodesResizedByDepth() {
return nodesResizedByDepth;
}
}

25
qa/backwards-5.0/build.gradle → core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttribute.java
View File

@ -17,18 +17,19 @@
* under the License.
*/
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
package org.apache.lucene.analysis.miscellaneous;
/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor. */
integTest {
includePackaged = true
}
import org.apache.lucene.util.Attribute;
integTestCluster {
numNodes = 4
numBwcNodes = 2
bwcVersion = project.wireCompatVersions[-1]
setting 'logger.org.elasticsearch', 'DEBUG'
}
/**
* Provides statistics useful for detecting duplicate sections of text
*/
public interface DuplicateSequenceAttribute extends Attribute {
/**
* @return The number of times this token has been seen previously as part
* of a sequence (counts to a max of 255)
*/
short getNumPriorUsesInASequence();
void setNumPriorUsesInASequence(short len);
}

53
core/src/main/java/org/apache/lucene/analysis/miscellaneous/DuplicateSequenceAttributeImpl.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
public class DuplicateSequenceAttributeImpl extends AttributeImpl implements DuplicateSequenceAttribute {
protected short numPriorUsesInASequence = 0;
@Override
public void clear() {
numPriorUsesInASequence = 0;
}
@Override
public void copyTo(AttributeImpl target) {
DuplicateSequenceAttributeImpl t = (DuplicateSequenceAttributeImpl) target;
t.numPriorUsesInASequence = numPriorUsesInASequence;
}
@Override
public short getNumPriorUsesInASequence() {
return numPriorUsesInASequence;
}
@Override
public void setNumPriorUsesInASequence(short len) {
numPriorUsesInASequence = len;
}
@Override
public void reflectWith(AttributeReflector reflector) {
reflector.reflect(DuplicateSequenceAttribute.class, "sequenceLength", numPriorUsesInASequence);
}
}

47
core/src/main/java/org/elasticsearch/Assertions.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch;
/**
* Provides a static final field that can be used to check if assertions are enabled. Since this field might be used elsewhere to check if
* assertions are enabled, if you are running with assertions enabled for specific packages or classes, you should enable assertions on this
* class too (e.g., {@code -ea org.elasticsearch.Assertions -ea org.elasticsearch.cluster.service.MasterService}).
*/
public final class Assertions {
private Assertions() {
}
public static final boolean ENABLED;
static {
boolean enabled = false;
/*
* If assertions are enabled, the following line will be evaluated and enabled will have the value true, otherwise when assertions
* are disabled enabled will have the value false.
*/
// noinspection ConstantConditions,AssertWithSideEffects
assert enabled = true;
// noinspection ConstantConditions
ENABLED = enabled;
}
}

3
core/src/main/java/org/elasticsearch/ElasticsearchException.java
View File

@ -877,8 +877,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
org.elasticsearch.transport.ReceiveTimeoutTransportException::new, 83, UNKNOWN_VERSION_ADDED),
NODE_DISCONNECTED_EXCEPTION(org.elasticsearch.transport.NodeDisconnectedException.class,
org.elasticsearch.transport.NodeDisconnectedException::new, 84, UNKNOWN_VERSION_ADDED),
ALREADY_EXPIRED_EXCEPTION(org.elasticsearch.index.AlreadyExpiredException.class,
org.elasticsearch.index.AlreadyExpiredException::new, 85, UNKNOWN_VERSION_ADDED),
// 85 used to be for AlreadyExpiredException
AGGREGATION_EXECUTION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationExecutionException.class,
org.elasticsearch.search.aggregations.AggregationExecutionException::new, 86, UNKNOWN_VERSION_ADDED),
// 87 used to be for MergeMappingException

34
core/src/main/java/org/elasticsearch/Version.java
View File

@ -74,15 +74,17 @@ public class Version implements Comparable<Version> {
public static final Version V_5_3_2 = new Version(V_5_3_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
public static final int V_5_4_0_ID = 5040099;
public static final Version V_5_4_0 = new Version(V_5_4_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_0);
public static final int V_5_5_0_ID_UNRELEASED = 5050099;
public static final Version V_5_5_0_UNRELEASED = new Version(V_5_5_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0);
public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
public static final Version V_6_0_0_alpha1_UNRELEASED =
new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0);
public static final int V_6_0_0_alpha2_ID_UNRELEASED = 6000002;
public static final Version V_6_0_0_alpha2_UNRELEASED =
new Version(V_6_0_0_alpha2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0);
public static final Version CURRENT = V_6_0_0_alpha2_UNRELEASED;
public static final int V_5_4_1_ID = 5040199;
public static final Version V_5_4_1 = new Version(V_5_4_1_ID, org.apache.lucene.util.Version.LUCENE_6_5_1);
public static final int V_5_5_0_ID = 5050099;
public static final Version V_5_5_0 = new Version(V_5_5_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_1);
public static final int V_6_0_0_alpha1_ID = 6000001;
public static final Version V_6_0_0_alpha1 =
new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
public static final int V_6_0_0_alpha2_ID = 6000002;
public static final Version V_6_0_0_alpha2 =
new Version(V_6_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
public static final Version CURRENT = V_6_0_0_alpha2;
// unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT)
@ -97,12 +99,14 @@ public class Version implements Comparable<Version> {
public static Version fromId(int id) {
switch (id) {
case V_6_0_0_alpha2_ID_UNRELEASED:
return V_6_0_0_alpha2_UNRELEASED;
case V_6_0_0_alpha1_ID_UNRELEASED:
return V_6_0_0_alpha1_UNRELEASED;
case V_5_5_0_ID_UNRELEASED:
return V_5_5_0_UNRELEASED;
case V_6_0_0_alpha2_ID:
return V_6_0_0_alpha2;
case V_6_0_0_alpha1_ID:
return V_6_0_0_alpha1;
case V_5_5_0_ID:
return V_5_5_0;
case V_5_4_1_ID:
return V_5_4_1;
case V_5_4_0_ID:
return V_5_4_0;
case V_5_3_2_ID:

4
core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
View File

@ -261,7 +261,7 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
type = in.readString();
id = in.readString();
version = in.readZLong();
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
seqNo = in.readZLong();
primaryTerm = in.readVLong();
} else {
@ -279,7 +279,7 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
out.writeString(type);
out.writeString(id);
out.writeZLong(version);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeZLong(seqNo);
out.writeVLong(primaryTerm);
}

4
core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
View File

@ -79,7 +79,7 @@ public class ClusterStateResponse extends ActionResponse {
super.readFrom(in);
clusterName = new ClusterName(in);
clusterState = ClusterState.readFrom(in, null);
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
totalCompressedSize = new ByteSizeValue(in);
} else {
// in a mixed cluster, if a pre 6.0 node processes the get cluster state
@ -95,7 +95,7 @@ public class ClusterStateResponse extends ActionResponse {
super.writeTo(out);
clusterName.writeTo(out);
clusterState.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
totalCompressedSize.writeTo(out);
}
}

2
core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
View File

@ -492,7 +492,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
for (int i = 0; i < size; i++) {
final String type = in.readString();
String source = in.readString();
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport
if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO change to 5.3.0 after backport
// we do not know the content type that comes from earlier versions so we autodetect and convert
source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
}

2
core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
View File

@ -308,7 +308,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
indicesOptions = IndicesOptions.readIndicesOptions(in);
type = in.readOptionalString();
source = in.readString();
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported
if (in.getVersion().before(Version.V_5_3_0)) {
// we do not know the format from earlier versions so convert if necessary
source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
}

6
core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.shards;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionResponse;
@ -34,7 +35,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.shard.ShardStateMetaData;
import java.io.IOException;
import java.util.ArrayList;
@ -165,7 +165,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
@Override
public void readFrom(StreamInput in) throws IOException {
node = new DiscoveryNode(in);
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().before(Version.V_6_0_0_alpha1)) {
// legacy version
in.readLong();
}
@ -179,7 +179,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
@Override
public void writeTo(StreamOutput out) throws IOException {
node.writeTo(out);
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
// legacy version
out.writeLong(-1L);
}

4
core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
View File

@ -104,7 +104,7 @@ public class ShardStats implements Streamable, Writeable, ToXContent {
statePath = in.readString();
dataPath = in.readString();
isCustomDataPath = in.readBoolean();
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
seqNoStats = in.readOptionalWriteable(SeqNoStats::new);
}
}
@ -117,7 +117,7 @@ public class ShardStats implements Streamable, Writeable, ToXContent {
out.writeString(statePath);
out.writeString(dataPath);
out.writeBoolean(isCustomDataPath);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeOptionalWriteable(seqNoStats);
}
}

6
core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
View File

@ -475,7 +475,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
cause = in.readString();
name = in.readString();
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
indexPatterns = in.readList(StreamInput::readString);
} else {
indexPatterns = Collections.singletonList(in.readString());
@ -487,7 +487,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
for (int i = 0; i < size; i++) {
final String type = in.readString();
String mappingSource = in.readString();
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported
if (in.getVersion().before(Version.V_5_3_0)) {
// we do not know the incoming type so convert it if needed
mappingSource =
XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource));
@ -512,7 +512,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
super.writeTo(out);
out.writeString(cause);
out.writeString(name);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeStringList(indexPatterns);
} else {
out.writeString(indexPatterns.size() > 0 ? indexPatterns.get(0) : "");

6
core/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
View File

@ -78,7 +78,7 @@ public class BulkItemRequest implements Streamable {
if (in.readBoolean()) {
primaryResponse = BulkItemResponse.readBulkItem(in);
}
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported
if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported
boolean ignoreOnReplica = in.readBoolean();
if (ignoreOnReplica == false && primaryResponse != null) {
assert primaryResponse.isFailed() == false : "expected no failure on the primary response";
@ -89,7 +89,7 @@ public class BulkItemRequest implements Streamable {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(id);
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported
if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported
// old nodes expect updated version and version type on the request
if (primaryResponse != null) {
request.version(primaryResponse.getVersion());
@ -102,7 +102,7 @@ public class BulkItemRequest implements Streamable {
DocWriteRequest.writeDocumentRequest(out, request);
}
out.writeOptionalStreamable(primaryResponse);
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO remove once backported
if (out.getVersion().before(Version.V_6_0_0_alpha1)) { // TODO remove once backported
if (primaryResponse != null) {
out.writeBoolean(primaryResponse.isFailed()
|| primaryResponse.getResponse().getResult() == DocWriteResponse.Result.NOOP);

4
core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
View File

@ -211,7 +211,7 @@ public class BulkItemResponse implements Streamable, StatusToXContentObject {
id = in.readOptionalString();
cause = in.readException();
status = ExceptionsHelper.status(cause);
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
seqNo = in.readZLong();
} else {
seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
@ -224,7 +224,7 @@ public class BulkItemResponse implements Streamable, StatusToXContentObject {
out.writeString(getType());
out.writeOptionalString(getId());
out.writeException(getCause());
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeZLong(getSeqNo());
}
}

6
core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View File

@ -54,6 +54,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndexClosedException;
import org.elasticsearch.ingest.IngestService;
@ -144,6 +145,11 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
// Attempt to create all the indices that we're going to need during the bulk before we start.
// Step 1: collect all the indices in the request
final Set<String> indices = bulkRequest.requests.stream()
// delete requests should not attempt to create the index (if the index does not
// exists), unless an external versioning is used
.filter(request -> request.opType() != DocWriteRequest.OpType.DELETE
|| request.versionType() == VersionType.EXTERNAL
|| request.versionType() == VersionType.EXTERNAL_GTE)
.map(DocWriteRequest::index)
.collect(Collectors.toSet());
/* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create

42
core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
View File

@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.delete.DeleteRequest;
@ -31,8 +30,8 @@ import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.replication.ReplicationOperation;
import org.elasticsearch.action.support.TransportActions;
import org.elasticsearch.action.support.replication.ReplicationOperation;
import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
import org.elasticsearch.action.support.replication.TransportWriteAction;
import org.elasticsearch.action.update.UpdateHelper;
@ -56,7 +55,6 @@ import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.get.GetResult;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Mapping;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.seqno.SequenceNumbersService;
@ -414,13 +412,6 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
FAILURE
}
static {
assert Version.CURRENT.minimumCompatibilityVersion().after(Version.V_6_0_0_alpha1_UNRELEASED) == false:
"Remove logic handling NoOp result from primary response; see TODO in replicaItemExecutionMode" +
" as the current minimum compatible version [" +
Version.CURRENT.minimumCompatibilityVersion() + "] is after 6.0";
}
/**
* Determines whether a bulk item request should be executed on the replica.
* @return {@link ReplicaItemExecutionMode#NORMAL} upon normal primary execution with no failures
@ -436,10 +427,11 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op
: ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication
} else {
// NOTE: write requests originating from pre-6.0 nodes can send a no-op operation to
// the replica; we ignore replication
// TODO: remove noOp result check from primary response, when pre-6.0 nodes are not supported
// we should return ReplicationItemExecutionMode.NORMAL instead
// TODO: once we know for sure that every operation that has been processed on the primary is assigned a seq#
// (i.e., all nodes on the cluster are on v6.0.0 or higher) we can use the existence of a seq# to indicate whether
// an operation should be processed or be treated as a noop. This means we could remove this method and the
// ReplicaItemExecutionMode enum and have a simple boolean check for seq != UNASSIGNED_SEQ_NO which will work for
// both failures and indexing operations.
return primaryResponse.getResponse().getResult() != DocWriteResponse.Result.NOOP
? ReplicaItemExecutionMode.NORMAL // execution successful on primary
: ReplicaItemExecutionMode.NOOP; // ignore replication
@ -454,6 +446,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
public static Translog.Location performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
Translog.Location location = null;
final long primaryTerm = request.primaryTerm();
for (int i = 0; i < request.items().length; i++) {
BulkItemRequest item = request.items()[i];
final Engine.Result operationResult;
@ -465,10 +458,12 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
switch (docWriteRequest.opType()) {
case CREATE:
case INDEX:
operationResult = executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, replica);
operationResult =
executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, primaryTerm, replica);
break;
case DELETE:
operationResult = executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, replica);
operationResult =
executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, primaryTerm, replica);
break;
default:
throw new IllegalStateException("Unexpected request operation type on replica: "
@ -536,14 +531,12 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
* Execute the given {@link IndexRequest} on a replica shard, throwing a
* {@link RetryOnReplicaException} if the operation needs to be re-tried.
*/
private static Engine.IndexResult executeIndexRequestOnReplica(
DocWriteResponse primaryResponse,
IndexRequest request,
IndexShard replica) throws IOException {
private static Engine.IndexResult executeIndexRequestOnReplica(DocWriteResponse primaryResponse, IndexRequest request,
long primaryTerm, IndexShard replica) throws IOException {
final Engine.Index operation;
try {
operation = prepareIndexOperationOnReplica(primaryResponse, request, replica);
operation = prepareIndexOperationOnReplica(primaryResponse, request, primaryTerm, replica);
} catch (MapperParsingException e) {
return new Engine.IndexResult(e, primaryResponse.getVersion(), primaryResponse.getSeqNo());
}
@ -561,6 +554,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
static Engine.Index prepareIndexOperationOnReplica(
DocWriteResponse primaryResponse,
IndexRequest request,
long primaryTerm,
IndexShard replica) {
final ShardId shardId = replica.shardId();
@ -573,7 +567,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
final VersionType versionType = request.versionType().versionTypeForReplicationAndRecovery();
assert versionType.validateVersionForWrites(version);
return replica.prepareIndexOnReplica(sourceToParse, seqNo, version, versionType,
return replica.prepareIndexOnReplica(sourceToParse, seqNo, primaryTerm, version, versionType,
request.getAutoGeneratedTimestamp(), request.isRetry());
}
@ -655,7 +649,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
}
private static Engine.DeleteResult executeDeleteRequestOnReplica(DocWriteResponse primaryResponse, DeleteRequest request,
IndexShard replica) throws Exception {
final long primaryTerm, IndexShard replica) throws Exception {
if (replica.indexSettings().isSingleType()) {
// We need to wait for the replica to have the mappings
Mapping update;
@ -675,7 +669,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
final long version = primaryResponse.getVersion();
assert versionType.validateVersionForWrites(version);
final Engine.Delete delete = replica.prepareDeleteOnReplica(request.type(), request.id(),
primaryResponse.getSeqNo(), request.primaryTerm(), version, versionType);
primaryResponse.getSeqNo(), primaryTerm, version, versionType);
return replica.delete(delete);
}

4
core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
View File

@ -78,7 +78,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
fields = in.readStringArray();
if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
indices = in.readStringArray();
indicesOptions = IndicesOptions.readIndicesOptions(in);
mergeResults = in.readBoolean();
@ -91,7 +91,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStringArray(fields);
if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
out.writeStringArray(indices);
indicesOptions.writeIndicesOptions(out);
out.writeBoolean(mergeResults);

4
core/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
View File

@ -86,7 +86,7 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
super.readFrom(in);
this.responseMap =
in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField);
if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
indexResponses = in.readList(FieldCapabilitiesIndexResponse::new);
} else {
indexResponses = Collections.emptyList();
@ -101,7 +101,7 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField);
if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
out.writeList(indexResponses);
}

65
core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
View File

@ -118,38 +118,45 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
for (Map.Entry<String, OriginalIndices> remoteIndices : remoteClusterIndices.entrySet()) {
String clusterAlias = remoteIndices.getKey();
OriginalIndices originalIndices = remoteIndices.getValue();
Transport.Connection connection = remoteClusterService.getConnection(remoteIndices.getKey());
FieldCapabilitiesRequest remoteRequest = new FieldCapabilitiesRequest();
remoteRequest.setMergeResults(false); // we need to merge on this node
remoteRequest.indicesOptions(originalIndices.indicesOptions());
remoteRequest.indices(originalIndices.indices());
remoteRequest.fields(request.fields());
transportService.sendRequest(connection, FieldCapabilitiesAction.NAME, remoteRequest, TransportRequestOptions.EMPTY,
new TransportResponseHandler<FieldCapabilitiesResponse>() {
@Override
public FieldCapabilitiesResponse newInstance() {
return new FieldCapabilitiesResponse();
}
// if we are connected this is basically a no-op, if we are not we try to connect in parallel in a non-blocking fashion
remoteClusterService.ensureConnected(clusterAlias, ActionListener.wrap(v -> {
Transport.Connection connection = remoteClusterService.getConnection(clusterAlias);
FieldCapabilitiesRequest remoteRequest = new FieldCapabilitiesRequest();
remoteRequest.setMergeResults(false); // we need to merge on this node
remoteRequest.indicesOptions(originalIndices.indicesOptions());
remoteRequest.indices(originalIndices.indices());
remoteRequest.fields(request.fields());
transportService.sendRequest(connection, FieldCapabilitiesAction.NAME, remoteRequest, TransportRequestOptions.EMPTY,
new TransportResponseHandler<FieldCapabilitiesResponse>() {
@Override
public void handleResponse(FieldCapabilitiesResponse response) {
for (FieldCapabilitiesIndexResponse res : response.getIndexResponses()) {
indexResponses.add(new FieldCapabilitiesIndexResponse(RemoteClusterAware.buildRemoteIndexName(clusterAlias,
res.getIndexName()), res.get()));
}
onResponse.run();
}
@Override
public FieldCapabilitiesResponse newInstance() {
return new FieldCapabilitiesResponse();
}
@Override
public void handleException(TransportException exp) {
onResponse.run();
}
@Override
public void handleResponse(FieldCapabilitiesResponse response) {
try {
for (FieldCapabilitiesIndexResponse res : response.getIndexResponses()) {
indexResponses.add(new FieldCapabilitiesIndexResponse(RemoteClusterAware.
buildRemoteIndexName(clusterAlias, res.getIndexName()), res.get()));
}
} finally {
onResponse.run();
}
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
});
@Override
public void handleException(TransportException exp) {
onResponse.run();
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
});
}, e -> onResponse.run()));
}
}

4
core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -523,7 +523,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
id = in.readOptionalString();
routing = in.readOptionalString();
parent = in.readOptionalString();
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().before(Version.V_6_0_0_alpha1)) {
in.readOptionalString(); // timestamp
in.readOptionalWriteable(TimeValue::new); // ttl
}
@ -548,7 +548,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
out.writeOptionalString(id);
out.writeOptionalString(routing);
out.writeOptionalString(parent);
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
// Serialize a fake timestamp. 5.x expect this value to be set by the #process method so we can't use null.
// On the other hand, indices created on 5.x do not index the timestamp field. Therefore passing a 0 (or any value) for
// the transport layer OK as it will be ignored.

48
core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java
View File

@ -23,6 +23,9 @@ import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.ArrayList;
@ -31,7 +34,7 @@ import java.util.List;
import static org.elasticsearch.action.ValidateActions.addValidationError;
public class ClearScrollRequest extends ActionRequest {
public class ClearScrollRequest extends ActionRequest implements ToXContentObject {
private List<String> scrollIds;
@ -83,4 +86,47 @@ public class ClearScrollRequest extends ActionRequest {
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startArray("scroll_id");
for (String scrollId : scrollIds) {
builder.value(scrollId);
}
builder.endArray();
builder.endObject();
return builder;
}
public void fromXContent(XContentParser parser) throws IOException {
scrollIds = null;
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("Malformed content, must start with an object");
} else {
XContentParser.Token token;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if ("scroll_id".equals(currentFieldName)){
if (token == XContentParser.Token.START_ARRAY) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token.isValue() == false) {
throw new IllegalArgumentException("scroll_id array element should only contain scroll_id");
}
addScrollId(parser.text());
}
} else {
if (token.isValue() == false) {
throw new IllegalArgumentException("scroll_id element should only contain scroll_id");
}
addScrollId(parser.text());
}
} else {
throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
+ "] in request body or parameter is of the wrong type[" + token + "] ");
}
}
}
}
}

31
core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
View File

@ -20,19 +20,34 @@
package org.elasticsearch.action.search;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
import static org.elasticsearch.rest.RestStatus.OK;
public class ClearScrollResponse extends ActionResponse implements StatusToXContentObject {
private static final ParseField SUCCEEDED = new ParseField("succeeded");
private static final ParseField NUMFREED = new ParseField("num_freed");
private static final ConstructingObjectParser<ClearScrollResponse, Void> PARSER = new ConstructingObjectParser<>("clear_scroll",
true, a -> new ClearScrollResponse((boolean)a[0], (int)a[1]));
static {
PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SUCCEEDED, ObjectParser.ValueType.BOOLEAN);
PARSER.declareField(constructorArg(), (parser, context) -> parser.intValue(), NUMFREED, ObjectParser.ValueType.INT);
}
private boolean succeeded;
private int numFreed;
@ -67,12 +82,19 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Fields.SUCCEEDED, succeeded);
builder.field(Fields.NUMFREED, numFreed);
builder.field(SUCCEEDED.getPreferredName(), succeeded);
builder.field(NUMFREED.getPreferredName(), numFreed);
builder.endObject();
return builder;
}
/**
* Parse the clear scroll response body into a new {@link ClearScrollResponse} object
*/
public static ClearScrollResponse fromXContent(XContentParser parser) throws IOException {
return PARSER.apply(parser, null);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
@ -86,9 +108,4 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
out.writeBoolean(succeeded);
out.writeVInt(numFreed);
}
static final class Fields {
static final String SUCCEEDED = "succeeded";
static final String NUMFREED = "num_freed";
}
}

38
core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.search.collapse.CollapseBuilder;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.function.Function;
/**
@ -59,7 +60,7 @@ final class ExpandSearchPhase extends SearchPhase {
final SearchRequest searchRequest = context.getRequest();
return searchRequest.source() != null &&
searchRequest.source().collapse() != null &&
searchRequest.source().collapse().getInnerHit() != null;
searchRequest.source().collapse().getInnerHits().isEmpty() == false;
}
@Override
@ -67,6 +68,7 @@ final class ExpandSearchPhase extends SearchPhase {
if (isCollapseRequest() && searchResponse.getHits().getHits().length > 0) {
SearchRequest searchRequest = context.getRequest();
CollapseBuilder collapseBuilder = searchRequest.source().collapse();
final List<InnerHitBuilder> innerHitBuilders = collapseBuilder.getInnerHits();
MultiSearchRequest multiRequest = new MultiSearchRequest();
if (collapseBuilder.getMaxConcurrentGroupRequests() > 0) {
multiRequest.maxConcurrentSearchRequests(collapseBuilder.getMaxConcurrentGroupRequests());
@ -83,27 +85,31 @@ final class ExpandSearchPhase extends SearchPhase {
if (origQuery != null) {
groupQuery.must(origQuery);
}
SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(collapseBuilder.getInnerHit())
.query(groupQuery);
SearchRequest groupRequest = new SearchRequest(searchRequest.indices())
.types(searchRequest.types())
.source(sourceBuilder);
multiRequest.add(groupRequest);
for (InnerHitBuilder innerHitBuilder : innerHitBuilders) {
SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder)
.query(groupQuery);
SearchRequest groupRequest = new SearchRequest(searchRequest.indices())
.types(searchRequest.types())
.source(sourceBuilder);
multiRequest.add(groupRequest);
}
}
context.getSearchTransport().sendExecuteMultiSearch(multiRequest, context.getTask(),
ActionListener.wrap(response -> {
Iterator<MultiSearchResponse.Item> it = response.iterator();
for (SearchHit hit : searchResponse.getHits()) {
MultiSearchResponse.Item item = it.next();
if (item.isFailure()) {
context.onPhaseFailure(this, "failed to expand hits", item.getFailure());
return;
for (InnerHitBuilder innerHitBuilder : innerHitBuilders) {
MultiSearchResponse.Item item = it.next();
if (item.isFailure()) {
context.onPhaseFailure(this, "failed to expand hits", item.getFailure());
return;
}
SearchHits innerHits = item.getResponse().getHits();
if (hit.getInnerHits() == null) {
hit.setInnerHits(new HashMap<>(innerHitBuilders.size()));
}
hit.getInnerHits().put(innerHitBuilder.getName(), innerHits);
}
SearchHits innerHits = item.getResponse().getHits();
if (hit.getInnerHits() == null) {
hit.setInnerHits(new HashMap<>(1));
}
hit.getInnerHits().put(collapseBuilder.getInnerHit().getName(), innerHits);
}
context.executeNextPhase(this, nextPhaseFactory.apply(searchResponse));
}, context::onFailure)

114
core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
View File

@ -21,32 +21,46 @@ package org.elasticsearch.action.search;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.action.RestActions;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.ProfileShardResult;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.suggest.Suggest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
import static org.elasticsearch.search.internal.InternalSearchResponse.readInternalSearchResponse;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
/**
* A response of a search request.
*/
public class SearchResponse extends ActionResponse implements StatusToXContentObject {
private InternalSearchResponse internalResponse;
private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
private static final ParseField TOOK = new ParseField("took");
private static final ParseField TIMED_OUT = new ParseField("timed_out");
private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
private static final ParseField NUM_REDUCE_PHASES = new ParseField("num_reduce_phases");
private SearchResponseSections internalResponse;
private String scrollId;
@ -61,7 +75,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
public SearchResponse() {
}
public SearchResponse(InternalSearchResponse internalResponse, String scrollId, int totalShards, int successfulShards,
public SearchResponse(SearchResponseSections internalResponse, String scrollId, int totalShards, int successfulShards,
long tookInMillis, ShardSearchFailure[] shardFailures) {
this.internalResponse = internalResponse;
this.scrollId = scrollId;
@ -176,7 +190,8 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
*
* @return The profile results or an empty map
*/
@Nullable public Map<String, ProfileShardResult> getProfileResults() {
@Nullable
public Map<String, ProfileShardResult> getProfileResults() {
return internalResponse.profile();
}
@ -190,15 +205,15 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
if (scrollId != null) {
builder.field("_scroll_id", scrollId);
builder.field(SCROLL_ID.getPreferredName(), scrollId);
}
builder.field("took", tookInMillis);
builder.field("timed_out", isTimedOut());
builder.field(TOOK.getPreferredName(), tookInMillis);
builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
if (isTerminatedEarly() != null) {
builder.field("terminated_early", isTerminatedEarly());
builder.field(TERMINATED_EARLY.getPreferredName(), isTerminatedEarly());
}
if (getNumReducePhases() != 1) {
builder.field("num_reduce_phases", getNumReducePhases());
builder.field(NUM_REDUCE_PHASES.getPreferredName(), getNumReducePhases());
}
RestActions.buildBroadcastShardsHeader(builder, params, getTotalShards(), getSuccessfulShards(), getFailedShards(),
getShardFailures());
@ -206,10 +221,89 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
return builder;
}
public static SearchResponse fromXContent(XContentParser parser) throws IOException {
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
XContentParser.Token token;
String currentFieldName = null;
SearchHits hits = null;
Aggregations aggs = null;
Suggest suggest = null;
SearchProfileShardResults profile = null;
boolean timedOut = false;
Boolean terminatedEarly = null;
int numReducePhases = 1;
long tookInMillis = -1;
int successfulShards = -1;
int totalShards = -1;
String scrollId = null;
List<ShardSearchFailure> failures = new ArrayList<>();
while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if (SCROLL_ID.match(currentFieldName)) {
scrollId = parser.text();
} else if (TOOK.match(currentFieldName)) {
tookInMillis = parser.longValue();
} else if (TIMED_OUT.match(currentFieldName)) {
timedOut = parser.booleanValue();
} else if (TERMINATED_EARLY.match(currentFieldName)) {
terminatedEarly = parser.booleanValue();
} else if (NUM_REDUCE_PHASES.match(currentFieldName)) {
numReducePhases = parser.intValue();
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (SearchHits.Fields.HITS.equals(currentFieldName)) {
hits = SearchHits.fromXContent(parser);
} else if (Aggregations.AGGREGATIONS_FIELD.equals(currentFieldName)) {
aggs = Aggregations.fromXContent(parser);
} else if (Suggest.NAME.equals(currentFieldName)) {
suggest = Suggest.fromXContent(parser);
} else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) {
profile = SearchProfileShardResults.fromXContent(parser);
} else if (RestActions._SHARDS_FIELD.match(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if (RestActions.FAILED_FIELD.match(currentFieldName)) {
parser.intValue(); // we don't need it but need to consume it
} else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName)) {
successfulShards = parser.intValue();
} else if (RestActions.TOTAL_FIELD.match(currentFieldName)) {
totalShards = parser.intValue();
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (RestActions.FAILURES_FIELD.match(currentFieldName)) {
while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
failures.add(ShardSearchFailure.fromXContent(parser));
}
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else {
throwUnknownToken(token, parser.getTokenLocation());
}
}
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
}
}
SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly,
profile, numReducePhases);
return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, tookInMillis,
failures.toArray(new ShardSearchFailure[failures.size()]));
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
internalResponse = readInternalSearchResponse(in);
internalResponse = new InternalSearchResponse(in);
totalShards = in.readVInt();
successfulShards = in.readVInt();
int size = in.readVInt();

122
core/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java Normal file
View File

@ -0,0 +1,122 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.search;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.profile.ProfileShardResult;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.suggest.Suggest;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
/**
* Base class that holds the various sections which a search response is
* composed of (hits, aggs, suggestions etc.) and allows to retrieve them.
*
* The reason why this class exists is that the high level REST client uses its own classes
* to parse aggregations into, which are not serializable. This is the common part that can be
* shared between core and client.
*/
public class SearchResponseSections implements ToXContent {
protected final SearchHits hits;
protected final Aggregations aggregations;
protected final Suggest suggest;
protected final SearchProfileShardResults profileResults;
protected final boolean timedOut;
protected final Boolean terminatedEarly;
protected final int numReducePhases;
public SearchResponseSections(SearchHits hits, Aggregations aggregations, Suggest suggest, boolean timedOut, Boolean terminatedEarly,
SearchProfileShardResults profileResults, int numReducePhases) {
this.hits = hits;
this.aggregations = aggregations;
this.suggest = suggest;
this.profileResults = profileResults;
this.timedOut = timedOut;
this.terminatedEarly = terminatedEarly;
this.numReducePhases = numReducePhases;
}
public final boolean timedOut() {
return this.timedOut;
}
public final Boolean terminatedEarly() {
return this.terminatedEarly;
}
public final SearchHits hits() {
return hits;
}
public final Aggregations aggregations() {
return aggregations;
}
public final Suggest suggest() {
return suggest;
}
/**
* Returns the number of reduce phases applied to obtain this search response
*/
public final int getNumReducePhases() {
return numReducePhases;
}
/**
* Returns the profile results for this search response (including all shards).
* An empty map is returned if profiling was not enabled
*
* @return Profile results
*/
public final Map<String, ProfileShardResult> profile() {
if (profileResults == null) {
return Collections.emptyMap();
}
return profileResults.getShardResults();
}
@Override
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
hits.toXContent(builder, params);
if (aggregations != null) {
aggregations.toXContent(builder, params);
}
if (suggest != null) {
suggest.toXContent(builder, params);
}
if (profileResults != null) {
profileResults.toXContent(builder, params);
}
return builder;
}
protected void writeTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
}

40
core/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
View File

@ -24,6 +24,9 @@ import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskId;
@ -33,7 +36,7 @@ import java.util.Objects;
import static org.elasticsearch.action.ValidateActions.addValidationError;
public class SearchScrollRequest extends ActionRequest {
public class SearchScrollRequest extends ActionRequest implements ToXContentObject {
private String scrollId;
private Scroll scroll;
@ -145,4 +148,39 @@ public class SearchScrollRequest extends ActionRequest {
return "scrollId[" + scrollId + "], scroll[" + scroll + "]";
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field("scroll_id", scrollId);
if (scroll != null) {
builder.field("scroll", scroll.keepAlive().getStringRep());
}
builder.endObject();
return builder;
}
/**
* Parse a search scroll request from a request body provided through the REST layer.
* Values that are already be set and are also found while parsing will be overridden.
*/
public void fromXContent(XContentParser parser) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("Malformed content, must start with an object");
} else {
XContentParser.Token token;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
scrollId(parser.text());
} else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
} else {
throw new IllegalArgumentException("Unknown parameter [" + currentFieldName
+ "] in request body or parameter is of the wrong type[" + token + "] ");
}
}
}
}
}

18
core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
View File

@ -38,7 +38,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@ -52,7 +51,6 @@ import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
@ -179,8 +177,8 @@ public abstract class TransportReplicationAction<
Request shardRequest, IndexShard primary) throws Exception;
/**
* Synchronous replica operation on nodes with replica copies. This is done under the lock form
* {@link IndexShard#acquireReplicaOperationLock(long, ActionListener, String)}
* Synchronously execute the specified replica operation. This is done under a permit from
* {@link IndexShard#acquireReplicaOperationPermit(long, ActionListener, String)}.
*
* @param shardRequest the request to the replica shard
* @param replica the replica shard to perform the operation on
@ -584,7 +582,7 @@ public abstract class TransportReplicationAction<
throw new ShardNotFoundException(this.replica.shardId(), "expected aID [{}] but found [{}]", targetAllocationID,
actualAllocationId);
}
replica.acquireReplicaOperationLock(request.primaryTerm, this, executor);
replica.acquireReplicaOperationPermit(request.primaryTerm, this, executor);
}
/**
@ -921,7 +919,7 @@ public abstract class TransportReplicationAction<
}
};
indexShard.acquirePrimaryOperationLock(onAcquired, executor);
indexShard.acquirePrimaryOperationPermit(onAcquired, executor);
}
class ShardReference implements Releasable {
@ -1013,7 +1011,7 @@ public abstract class TransportReplicationAction<
@Override
public void readFrom(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
super.readFrom(in);
localCheckpoint = in.readZLong();
allocationId = in.readString();
@ -1024,7 +1022,7 @@ public abstract class TransportReplicationAction<
@Override
public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
super.writeTo(out);
out.writeZLong(localCheckpoint);
out.writeString(allocationId);
@ -1193,7 +1191,7 @@ public abstract class TransportReplicationAction<
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
globalCheckpoint = in.readZLong();
}
}
@ -1201,7 +1199,7 @@ public abstract class TransportReplicationAction<
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeZLong(globalCheckpoint);
}
}

6
core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
View File

@ -44,10 +44,8 @@ import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.index.mapper.RoutingFieldMapper;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.lookup.SourceLookup;
@ -301,8 +299,8 @@ public class UpdateHelper extends AbstractComponent {
private Map<String, Object> executeScript(Script script, Map<String, Object> ctx) {
try {
if (scriptService != null) {
CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.UPDATE);
ExecutableScript executableScript = scriptService.executable(compiledScript, script.getParams());
ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.UPDATE_CONTEXT);
ExecutableScript executableScript = factory.newInstance(script.getParams());
executableScript.setNextVar(ContextFields.CTX, ctx);
executableScript.run();
}

4
core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java
View File

@ -138,7 +138,7 @@ public class ClusterBlock implements Streamable, ToXContent {
retryable = in.readBoolean();
disableStatePersistence = in.readBoolean();
status = RestStatus.readFrom(in);
if (in.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
allowReleaseResources = in.readBoolean();
} else {
allowReleaseResources = false;
@ -156,7 +156,7 @@ public class ClusterBlock implements Streamable, ToXContent {
out.writeBoolean(retryable);
out.writeBoolean(disableStatePersistence);
RestStatus.writeTo(out, status);
if (out.getVersion().onOrAfter(Version.V_5_5_0_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
out.writeBoolean(allowReleaseResources);
}
}

4
core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View File

@ -210,7 +210,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
Builder builder = new Builder(in.readString());
builder.order(in.readInt());
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
builder.patterns(in.readList(StreamInput::readString));
} else {
builder.patterns(Collections.singletonList(in.readString()));
@ -245,7 +245,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeInt(order);
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
out.writeStringList(patterns);
} else {
out.writeString(patterns.size() > 0 ? patterns.get(0) : "");

4
core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -196,7 +196,7 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
source().writeTo(out);
// routing
out.writeBoolean(routing().required());
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
// timestamp
out.writeBoolean(false); // enabled
out.writeString(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format());
@ -233,7 +233,7 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
source = CompressedXContent.readCompressedString(in);
// routing
routing = new Routing(in.readBoolean());
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().before(Version.V_6_0_0_alpha1)) {
// timestamp
boolean enabled = in.readBoolean();
if (enabled) {

15
core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.routing;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNodes;
@ -177,10 +178,20 @@ public class OperationRouting extends AbstractComponent {
}
}
// if not, then use it as the index
int routingHash = Murmur3HashFunction.hash(preference);
if (nodes.getMinNodeVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
// The AllocationService lists shards in a fixed order based on nodes
// so earlier versions of this class would have a tendency to
// select the same node across different shardIds.
// Better overall balancing can be achieved if each shardId opts
// for a different element in the list by also incorporating the
// shard ID into the hash of the user-supplied preference key.
routingHash = 31 * routingHash + indexShard.shardId.hashCode();
}
if (awarenessAttributes.length == 0) {
return indexShard.activeInitializingShardsIt(Murmur3HashFunction.hash(preference));
return indexShard.activeInitializingShardsIt(routingHash);
} else {
return indexShard.preferAttributesActiveInitializingShardsIt(awarenessAttributes, nodes, Murmur3HashFunction.hash(preference));
return indexShard.preferAttributesActiveInitializingShardsIt(awarenessAttributes, nodes, routingHash);
}
}

5
core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
View File

@ -23,6 +23,7 @@ import com.carrotsearch.hppc.ObjectIntHashMap;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.Assertions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
@ -1018,9 +1019,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
* this method does nothing.
*/
public static boolean assertShardStats(RoutingNodes routingNodes) {
boolean run = false;
assert (run = true); // only run if assertions are enabled!
if (!run) {
if (!Assertions.ENABLED) {
return true;
}
int unassignedPrimaryCount = 0;

5
core/src/main/java/org/elasticsearch/cluster/service/MasterService.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.cluster.service;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.Assertions;
import org.elasticsearch.cluster.AckedClusterStateTaskListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
@ -666,9 +667,7 @@ public class MasterService extends AbstractLifecycleComponent {
assert clusterTasksResult.executionResults.size() == taskInputs.updateTasks.size()
: String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", taskInputs.updateTasks.size(),
taskInputs.updateTasks.size() == 1 ? "" : "s", clusterTasksResult.executionResults.size());
boolean assertsEnabled = false;
assert (assertsEnabled = true);
if (assertsEnabled) {
if (Assertions.ENABLED) {
ClusterTasksResult<Object> finalClusterTasksResult = clusterTasksResult;
taskInputs.updateTasks.forEach(updateTask -> {
assert finalClusterTasksResult.executionResults.containsKey(updateTask.task) :

5
core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
View File

@ -23,6 +23,7 @@ import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryFactory;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.Assertions;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.io.stream.NamedWriteable;
@ -58,9 +59,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
static {
// if asserts are enabled we run the debug statements even if they are not logged
// to prevent exceptions only present if debug enabled
boolean debug = false;
assert debug = true;
DEBUG = debug;
DEBUG = Assertions.ENABLED;
}
public static final double DATELINE = 180;

6
core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.common.lucene;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.elasticsearch.Assertions;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
@ -152,10 +153,7 @@ public final class ShardCoreKeyMap {
}
private synchronized boolean assertSize() {
// this is heavy and should only used in assertions
boolean assertionsEnabled = false;
assert assertionsEnabled = true;
if (assertionsEnabled == false) {
if (!Assertions.ENABLED) {
throw new AssertionError("only run this if assertions are enabled");
}
Collection<Set<IndexReader.CacheKey>> values = indexToCoreKey.values();

7
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.common.settings;
import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.transport.RemoteClusterAware;
import org.elasticsearch.action.search.TransportSearchAction;
import org.elasticsearch.action.support.AutoCreateIndex;
import org.elasticsearch.action.support.DestructiveOperations;
@ -88,6 +86,8 @@ import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.RemoteClusterAware;
import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportService;
@ -304,6 +304,8 @@ public final class ClusterSettings extends AbstractScopedSettings {
ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE,
ScriptService.TYPES_ALLOWED_SETTING,
ScriptService.CONTEXTS_ALLOWED_SETTING,
IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING,
IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY,
IndicesRequestCache.INDICES_CACHE_QUERY_SIZE,
@ -339,6 +341,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
ZenDiscovery.SEND_LEAVE_REQUEST_SETTING,
ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING,
ZenDiscovery.MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING,
ZenDiscovery.MAX_PENDING_CLUSTER_STATES_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT,

2
core/src/main/java/org/elasticsearch/common/settings/Settings.java
View File

@ -344,7 +344,7 @@ public final class Settings implements ToXContent {
final String setting,
final Boolean defaultValue,
final DeprecationLogger deprecationLogger) {
if (indexVersion.before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (indexVersion.before(Version.V_6_0_0_alpha1)) {
//Only emit a warning if the setting's value is not a proper boolean
final String value = get(setting, "false");
if (Booleans.isBoolean(value) == false) {

4
core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
View File

@ -78,7 +78,7 @@ public final class TransportAddress implements Writeable {
* {@link Version#V_5_0_2} as the hostString was not serialized
*/
public TransportAddress(StreamInput in, @Nullable String hostString) throws IOException {
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // bwc layer for 5.x where we had more than one transport address
if (in.getVersion().before(Version.V_6_0_0_alpha1)) { // bwc layer for 5.x where we had more than one transport address
final short i = in.readShort();
if(i != 1) { // we fail hard to ensure nobody tries to use some custom transport address impl even if that is difficult to add
throw new AssertionError("illegal transport ID from node of version: " + in.getVersion() + " got: " + i + " expected: 1");
@ -101,7 +101,7 @@ public final class TransportAddress implements Writeable {
@Override
public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
out.writeShort((short)1); // this maps to InetSocketTransportAddress in 5.x
}
byte[] bytes = address.getAddress().getAddress(); // 4 bytes (IPv4) or 16 bytes (IPv6)

10
core/src/main/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutor.java
View File

@ -91,19 +91,19 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
* Calculate task rate (λ), for a fixed number of tasks and time it took those tasks to be measured
*
* @param totalNumberOfTasks total number of tasks that were measured
* @param totalFrameFrameNanos nanoseconds during which the tasks were received
* @param totalFrameTaskNanos nanoseconds during which the tasks were received
* @return the rate of tasks in the system
*/
static double calculateLambda(final int totalNumberOfTasks, final long totalFrameFrameNanos) {
assert totalFrameFrameNanos > 0 : "cannot calculate for instantaneous tasks";
assert totalNumberOfTasks > 0 : "cannot calculate for no tasks";
static double calculateLambda(final int totalNumberOfTasks, final long totalFrameTaskNanos) {
assert totalFrameTaskNanos > 0 : "cannot calculate for instantaneous tasks, got: " + totalFrameTaskNanos;
assert totalNumberOfTasks > 0 : "cannot calculate for no tasks, got: " + totalNumberOfTasks;
// There is no set execution time, instead we adjust the time window based on the
// number of completed tasks, so there is no background thread required to update the
// queue size at a regular interval. This means we need to calculate our λ by the
// total runtime, rather than a fixed interval.
// λ = total tasks divided by measurement time
return (double) totalNumberOfTasks / totalFrameFrameNanos;
return (double) totalNumberOfTasks / totalFrameTaskNanos;
}
/**

5
core/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.util.concurrent;
import org.elasticsearch.Assertions;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.index.engine.EngineException;
@ -35,9 +36,7 @@ public class ReleasableLock implements Releasable {
public ReleasableLock(Lock lock) {
this.lock = lock;
boolean useHoldingThreads = false;
assert (useHoldingThreads = true);
if (useHoldingThreads) {
if (Assertions.ENABLED) {
holdingThreads = new ThreadLocal<>();
} else {
holdingThreads = null;

4
core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
View File

@ -395,6 +395,7 @@ public final class ObjectParser<Value, Context> extends AbstractObjectParser<Val
FLOAT(VALUE_NUMBER, VALUE_STRING),
FLOAT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
DOUBLE(VALUE_NUMBER, VALUE_STRING),
DOUBLE_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
LONG(VALUE_NUMBER, VALUE_STRING),
LONG_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
INT(VALUE_NUMBER, VALUE_STRING),
@ -411,7 +412,8 @@ public final class ObjectParser<Value, Context> extends AbstractObjectParser<Val
OBJECT_OR_STRING(START_OBJECT, VALUE_STRING),
OBJECT_ARRAY_BOOLEAN_OR_STRING(START_OBJECT, START_ARRAY, VALUE_BOOLEAN, VALUE_STRING),
OBJECT_ARRAY_OR_STRING(START_OBJECT, START_ARRAY, VALUE_STRING),
VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING);
VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING),
VALUE_OBJECT_ARRAY(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING, START_OBJECT, START_ARRAY);
private final EnumSet<XContentParser.Token> tokens;

8
core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
View File

@ -111,10 +111,9 @@ public final class XContentParserUtils {
}
/**
* This method expects that the current token is a {@code XContentParser.Token.FIELD_NAME} and
* that the current field name is the concatenation of a type, delimiter and name (ex: terms#foo
* where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry}, "#" is
* the delimiter and "foo" the name of the object to parse).
* This method expects that the current field name is the concatenation of a type, a delimiter and a name
* (ex: terms#foo where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry},
* "#" is the delimiter and "foo" the name of the object to parse).
*
* The method splits the field's name to extract the type and name and then parses the object
* using the {@link XContentParser#namedObject(Class, String, Object)} method.
@ -128,7 +127,6 @@ public final class XContentParserUtils {
* from the field's name
*/
public static <T> T parseTypedKeysObject(XContentParser parser, String delimiter, Class<T> objectClass) throws IOException {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
String currentFieldName = parser.currentName();
if (Strings.hasLength(currentFieldName)) {
int position = currentFieldName.indexOf(delimiter);

93
core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
View File

@ -23,8 +23,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
@ -60,61 +60,53 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
public class PublishClusterStateAction extends AbstractComponent {
public static final String SEND_ACTION_NAME = "internal:discovery/zen/publish/send";
public static final String COMMIT_ACTION_NAME = "internal:discovery/zen/publish/commit";
public static final String SETTINGS_MAX_PENDING_CLUSTER_STATES = "discovery.zen.publish.max_pending_cluster_states";
public interface IncomingClusterStateListener {
public interface NewPendingClusterStateListener {
/**
* called when a new incoming cluster state has been received.
* Should validate the incoming state and throw an exception if it's not a valid successor state.
*/
void onIncomingClusterState(ClusterState incomingState);
/** a new cluster state has been committed and is ready to process via {@link #pendingStatesQueue()} */
void onNewClusterState(String reason);
/**
* called when a cluster state has been committed and is ready to be processed
*/
void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener);
}
private final TransportService transportService;
private final NamedWriteableRegistry namedWriteableRegistry;
private final Supplier<ClusterState> clusterStateSupplier;
private final NewPendingClusterStateListener newPendingClusterStatelistener;
private final IncomingClusterStateListener incomingClusterStateListener;
private final DiscoverySettings discoverySettings;
private final ClusterName clusterName;
private final PendingClusterStatesQueue pendingStatesQueue;
public PublishClusterStateAction(
Settings settings,
TransportService transportService,
NamedWriteableRegistry namedWriteableRegistry,
Supplier<ClusterState> clusterStateSupplier,
NewPendingClusterStateListener listener,
DiscoverySettings discoverySettings,
ClusterName clusterName) {
IncomingClusterStateListener incomingClusterStateListener,
DiscoverySettings discoverySettings) {
super(settings);
this.transportService = transportService;
this.namedWriteableRegistry = namedWriteableRegistry;
this.clusterStateSupplier = clusterStateSupplier;
this.newPendingClusterStatelistener = listener;
this.incomingClusterStateListener = incomingClusterStateListener;
this.discoverySettings = discoverySettings;
this.clusterName = clusterName;
this.pendingStatesQueue = new PendingClusterStatesQueue(logger, settings.getAsInt(SETTINGS_MAX_PENDING_CLUSTER_STATES, 25));
transportService.registerRequestHandler(SEND_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.SAME, false, false,
new SendClusterStateRequestHandler());
transportService.registerRequestHandler(COMMIT_ACTION_NAME, CommitClusterStateRequest::new, ThreadPool.Names.SAME, false, false,
new CommitClusterStateRequestHandler());
}
public PendingClusterStatesQueue pendingStatesQueue() {
return pendingStatesQueue;
}
/**
* publishes a cluster change event to other nodes. if at least minMasterNodes acknowledge the change it is committed and will
* be processed by the master and the other nodes.
@ -387,7 +379,7 @@ public class PublishClusterStateAction extends AbstractComponent {
final ClusterState incomingState;
// If true we received full cluster state - otherwise diffs
if (in.readBoolean()) {
incomingState = ClusterState.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
incomingState = ClusterState.readFrom(in, transportService.getLocalNode());
logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(),
request.bytes().length());
} else if (lastSeenClusterState != null) {
@ -399,10 +391,7 @@ public class PublishClusterStateAction extends AbstractComponent {
logger.debug("received diff for but don't have any local cluster state - requesting full state");
throw new IncompatibleClusterStateVersionException("have no local cluster state");
}
// sanity check incoming state
validateIncomingState(incomingState, lastSeenClusterState);
pendingStatesQueue.addPending(incomingState);
incomingClusterStateListener.onIncomingClusterState(incomingState);
lastSeenClusterState = incomingState;
}
} finally {
@ -411,56 +400,22 @@ public class PublishClusterStateAction extends AbstractComponent {
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
// package private for testing
/**
* does simple sanity check of the incoming cluster state. Throws an exception on rejections.
*/
void validateIncomingState(ClusterState incomingState, ClusterState lastSeenClusterState) {
final ClusterName incomingClusterName = incomingState.getClusterName();
if (!incomingClusterName.equals(this.clusterName)) {
logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]",
incomingState.nodes().getMasterNode(), incomingClusterName);
throw new IllegalStateException("received state from a node that is not part of the cluster");
}
final ClusterState clusterState = clusterStateSupplier.get();
if (clusterState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) {
logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen",
incomingState.nodes().getMasterNode());
throw new IllegalStateException("received state with a local node that does not match the current local node");
}
if (ZenDiscovery.shouldIgnoreOrRejectNewClusterState(logger, clusterState, incomingState)) {
String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
logger.warn(message);
throw new IllegalStateException(message);
}
}
protected void handleCommitRequest(CommitClusterStateRequest request, final TransportChannel channel) {
final ClusterState state = pendingStatesQueue.markAsCommitted(request.stateUUID,
new PendingClusterStatesQueue.StateProcessedListener() {
incomingClusterStateListener.onClusterStateCommitted(request.stateUUID, new ActionListener<Void>() {
@Override
public void onNewClusterStateProcessed() {
public void onResponse(Void ignore) {
try {
// send a response to the master to indicate that this cluster state has been processed post committing it.
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (Exception e) {
logger.debug("failed to send response on cluster state processed", e);
onNewClusterStateFailed(e);
onFailure(e);
}
}
@Override
public void onNewClusterStateFailed(Exception e) {
public void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (Exception inner) {
@ -469,10 +424,6 @@ public class PublishClusterStateAction extends AbstractComponent {
}
}
});
if (state != null) {
newPendingClusterStatelistener.onNewClusterState("master " + state.nodes().getMasterNode() +
" committed version [" + state.version() + "]");
}
}
private class SendClusterStateRequestHandler implements TransportRequestHandler<BytesTransportRequest> {

93
core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
View File

@ -25,6 +25,7 @@ import org.apache.logging.log4j.util.Supplier;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
@ -56,6 +57,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.DiscoveryStats;
import org.elasticsearch.discovery.zen.PublishClusterStateAction.IncomingClusterStateListener;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.EmptyTransportResponseHandler;
import org.elasticsearch.transport.TransportChannel;
@ -82,7 +84,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider {
public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider, IncomingClusterStateListener {
public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
@ -104,6 +106,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
Property.NodeScope);
public static final Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope);
public static final Setting<Integer> MAX_PENDING_CLUSTER_STATES_SETTING =
Setting.intSetting("discovery.zen.publish.max_pending_cluster_states", 25, 1, Property.NodeScope);
public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
@ -139,6 +143,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
private final JoinThreadControl joinThreadControl;
private final PendingClusterStatesQueue pendingStatesQueue;
private final NodeJoinController nodeJoinController;
private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
@ -197,16 +203,15 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
this.masterFD.addListener(new MasterNodeFailureListener());
this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterName);
this.nodesFD.addListener(new NodeFaultDetectionListener());
this.pendingStatesQueue = new PendingClusterStatesQueue(logger, MAX_PENDING_CLUSTER_STATES_SETTING.get(settings));
this.publishClusterState =
new PublishClusterStateAction(
settings,
transportService,
namedWriteableRegistry,
this::clusterState,
new NewPendingClusterStateListener(),
discoverySettings,
clusterName);
this,
discoverySettings);
this.membership = new MembershipAction(settings, transportService, new MembershipListener());
this.joinThreadControl = new JoinThreadControl();
@ -311,7 +316,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
throw new FailedToCommitClusterStateException("state was mutated while calculating new CS update");
}
publishClusterState.pendingStatesQueue().addPending(newState);
pendingStatesQueue.addPending(newState);
try {
publishClusterState.publish(clusterChangedEvent, electMaster.minimumMasterNodes(), ackListener);
@ -321,7 +326,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
newState.version(), electMaster.minimumMasterNodes());
synchronized (stateMutex) {
publishClusterState.pendingStatesQueue().failAllStatesAndClear(
pendingStatesQueue.failAllStatesAndClear(
new ElasticsearchException("failed to publish cluster state"));
rejoin("zen-disco-failed-to-publish");
@ -332,7 +337,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
final DiscoveryNode localNode = newState.getNodes().getLocalNode();
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean processedOrFailed = new AtomicBoolean();
publishClusterState.pendingStatesQueue().markAsCommitted(newState.stateUUID(),
pendingStatesQueue.markAsCommitted(newState.stateUUID(),
new PendingClusterStatesQueue.StateProcessedListener() {
@Override
public void onNewClusterStateProcessed() {
@ -391,7 +396,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
@Override
public DiscoveryStats stats() {
PendingClusterStateStats queueStats = publishClusterState.pendingStatesQueue().stats();
PendingClusterStateStats queueStats = pendingStatesQueue.stats();
return new DiscoveryStats(queueStats);
}
@ -409,11 +414,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
// used for testing
public ClusterState[] pendingClusterStates() {
return publishClusterState.pendingStatesQueue().pendingClusterStates();
return pendingStatesQueue.pendingClusterStates();
}
PendingClusterStatesQueue pendingClusterStatesQueue() {
return publishClusterState.pendingStatesQueue();
return pendingStatesQueue;
}
/**
@ -703,7 +708,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
synchronized (stateMutex) {
if (localNodeMaster() == false && masterNode.equals(committedState.get().nodes().getMasterNode())) {
// flush any pending cluster states from old master, so it will not be set as master again
publishClusterState.pendingStatesQueue().failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason));
pendingStatesQueue.failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason));
rejoin("master left (reason = " + reason + ")");
}
}
@ -713,7 +718,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
boolean processNextCommittedClusterState(String reason) {
assert Thread.holdsLock(stateMutex);
final ClusterState newClusterState = publishClusterState.pendingStatesQueue().getNextClusterStateToProcess();
final ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess();
final ClusterState currentState = committedState.get();
final ClusterState adaptedNewClusterState;
// all pending states have been processed
@ -742,7 +747,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
}
} catch (Exception e) {
try {
publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e);
pendingStatesQueue.markAsFailed(newClusterState, e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner);
@ -811,7 +816,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
try {
publishClusterState.pendingStatesQueue().markAsProcessed(newClusterState);
pendingStatesQueue.markAsProcessed(newClusterState);
} catch (Exception e) {
onFailure(source, e);
}
@ -823,7 +828,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
try {
// TODO: use cluster state uuid instead of full cluster state so that we don't keep reference to CS around
// for too long.
publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e);
pendingStatesQueue.markAsFailed(newClusterState, e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner);
@ -1066,16 +1071,64 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
}
}
private class NewPendingClusterStateListener implements PublishClusterStateAction.NewPendingClusterStateListener {
@Override
public void onIncomingClusterState(ClusterState incomingState) {
validateIncomingState(logger, incomingState, committedState.get());
pendingStatesQueue.addPending(incomingState);
}
@Override
public void onNewClusterState(String reason) {
@Override
public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) {
final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID,
new PendingClusterStatesQueue.StateProcessedListener() {
@Override
public void onNewClusterStateProcessed() {
processedListener.onResponse(null);
}
@Override
public void onNewClusterStateFailed(Exception e) {
processedListener.onFailure(e);
}
});
if (state != null) {
synchronized (stateMutex) {
processNextCommittedClusterState(reason);
processNextCommittedClusterState("master " + state.nodes().getMasterNode() +
" committed version [" + state.version() + "]");
}
}
}
/**
* does simple sanity check of the incoming cluster state. Throws an exception on rejections.
*/
static void validateIncomingState(Logger logger, ClusterState incomingState, ClusterState lastState) {
final ClusterName incomingClusterName = incomingState.getClusterName();
if (!incomingClusterName.equals(lastState.getClusterName())) {
logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]",
incomingState.nodes().getMasterNode(), incomingClusterName);
throw new IllegalStateException("received state from a node that is not part of the cluster");
}
if (lastState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) {
logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen",
incomingState.nodes().getMasterNode());
throw new IllegalStateException("received state with a local node that does not match the current local node");
}
if (shouldIgnoreOrRejectNewClusterState(logger, lastState, incomingState)) {
String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
logger.warn(message);
throw new IllegalStateException(message);
}
}
private class MembershipListener implements MembershipAction.MembershipListener {
@Override
public void onJoin(DiscoveryNode node, MembershipAction.JoinCallback callback) {

4
core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
View File

@ -290,7 +290,7 @@ public class TransportNodesListGatewayStartedShards extends
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (in.getVersion().before(Version.V_6_0_0_alpha1)) {
// legacy version
in.readLong();
}
@ -304,7 +304,7 @@ public class TransportNodesListGatewayStartedShards extends
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
// legacy version
out.writeLong(-1L);
}

91
core/src/main/java/org/elasticsearch/index/AlreadyExpiredException.java
View File

@ -1,91 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException;
import java.io.IOException;
public class AlreadyExpiredException extends ElasticsearchException implements IgnoreOnRecoveryEngineException {
private String index;
private String type;
private String id;
private final long timestamp;
private final long ttl;
private final long now;
public AlreadyExpiredException(String index, String type, String id, long timestamp, long ttl, long now) {
super("already expired [" + index + "]/[" + type + "]/[" + id + "] due to expire at [" + (timestamp + ttl) + "] and was processed at [" + now + "]");
this.setIndex(index);
this.type = type;
this.id = id;
this.timestamp = timestamp;
this.ttl = ttl;
this.now = now;
}
public String index() {
return index;
}
public String type() {
return type;
}
public String id() {
return id;
}
public long timestamp() {
return timestamp;
}
public long ttl() {
return ttl;
}
public long now() {
return now;
}
public AlreadyExpiredException(StreamInput in) throws IOException{
super(in);
index = in.readOptionalString();
type = in.readOptionalString();
id = in.readOptionalString();
timestamp = in.readLong();
ttl = in.readLong();
now = in.readLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalString(index);
out.writeOptionalString(type);
out.writeOptionalString(id);
out.writeLong(timestamp);
out.writeLong(ttl);
out.writeLong(now);
}
}

5
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -612,11 +612,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
rescheduleFsyncTask(durability);
}
}
// update primary terms
for (final IndexShard shard : this.shards.values()) {
shard.updatePrimaryTerm(metadata.primaryTerm(shard.shardId().id()));
}
}
private void rescheduleFsyncTask(Translog.Durability durability) {

4
core/src/main/java/org/elasticsearch/index/IndexSortConfig.java
View File

@ -120,13 +120,13 @@ public final class IndexSortConfig {
.map((name) -> new FieldSortSpec(name))
.toArray(FieldSortSpec[]::new);
if (sortSpecs.length > 0 && indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
if (sortSpecs.length > 0 && indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_alpha1)) {
/**
* This index might be assigned to a node where the index sorting feature is not available
* (ie. versions prior to {@link Version.V_6_0_0_alpha1_UNRELEASED}) so we must fail here rather than later.
*/
throw new IllegalArgumentException("unsupported index.version.created:" + indexSettings.getIndexVersionCreated() +
", can't set index.sort on versions prior to " + Version.V_6_0_0_alpha1_UNRELEASED);
", can't set index.sort on versions prior to " + Version.V_6_0_0_alpha1);
}
if (INDEX_SORT_ORDER_SETTING.exists(settings)) {

Some files were not shown because too many files have changed in this diff Show More