diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index e836bd2fa26..67ebfd5c9bb 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -123,12 +123,19 @@ class BuildPlugin implements Plugin { } println " Random Testing Seed : ${project.testSeed}" - // enforce gradle version - GradleVersion minGradle = GradleVersion.version('3.3') - if (GradleVersion.current() < minGradle) { + // enforce Gradle version + final GradleVersion currentGradleVersion = GradleVersion.current(); + + final GradleVersion minGradle = GradleVersion.version('3.3') + if (currentGradleVersion < minGradle) { throw new GradleException("${minGradle} or above is required to build elasticsearch") } + final GradleVersion maxGradle = GradleVersion.version('4.2') + if (currentGradleVersion >= maxGradle) { + throw new GradleException("${maxGradle} or above is not compatible with the elasticsearch build") + } + // enforce Java version if (javaVersionEnum < minimumJava) { throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java new file mode 100644 index 00000000000..2cc1d4849d5 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.Header; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; + +import java.io.IOException; +import java.util.Collections; + +/** + * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Indices API. + * + * See Indices API on elastic.co + */ +public final class IndicesClient { + private final RestHighLevelClient restHighLevelClient; + + public IndicesClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } + + /** + * Deletes an index using the Delete Index API + *

+ * See + * Delete Index API on elastic.co + */ + public DeleteIndexResponse deleteIndex(DeleteIndexRequest deleteIndexRequest, Header... headers) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent, + Collections.emptySet(), headers); + } + + /** + * Asynchronously deletes an index using the Delete Index API + *

+ * See + * Delete Index API on elastic.co + */ + public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener listener, Header... headers) { + restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent, + listener, Collections.emptySet(), headers); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java old mode 100644 new mode 100755 index 7a95553c3c0..4da68e98e2d --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java @@ -29,6 +29,7 @@ import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; @@ -123,6 +124,17 @@ public final class Request { return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null); } + static Request deleteIndex(DeleteIndexRequest deleteIndexRequest) { + String endpoint = endpoint(deleteIndexRequest.indices(), Strings.EMPTY_ARRAY, ""); + + Params parameters = Params.builder(); + parameters.withTimeout(deleteIndexRequest.timeout()); + parameters.withMasterTimeout(deleteIndexRequest.masterNodeTimeout()); + parameters.withIndicesOptions(deleteIndexRequest.indicesOptions()); + + return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null); + } + static Request info() { return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null); } @@ -449,6 +461,10 @@ public final class Request { return this; } + Params withMasterTimeout(TimeValue masterTimeout) { + return putParam("master_timeout", masterTimeout); + } + Params withParent(String parent) { return putParam("parent", parent); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java old mode 100644 new mode 100755 index 25697abb82e..bc3538930d3 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -176,6 +176,8 @@ public class RestHighLevelClient implements Closeable { private final NamedXContentRegistry registry; private final CheckedConsumer doClose; + private final IndicesClient indicesClient = new IndicesClient(this); + /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the * {@link RestClient} to be used to perform requests. @@ -220,6 +222,15 @@ public class RestHighLevelClient implements Closeable { doClose.accept(client); } + /** + * Provides an {@link IndicesClient} which can be used to access the Indices API. + * + * See Indices API on elastic.co + */ + public IndicesClient indices() { + return indicesClient; + } + /** * Executes a bulk request using the Bulk API * @@ -327,7 +338,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Deletes a document by id using the Delete api + * Deletes a document by id using the Delete API * * See Delete API on elastic.co */ @@ -337,7 +348,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Asynchronously deletes a document by id using the Delete api + * Asynchronously deletes a document by id using the Delete API * * See Delete API on elastic.co */ @@ -347,7 +358,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Executes a search using the Search api + * Executes a search using the Search API * * See Search API on elastic.co */ @@ -356,7 +367,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Asynchronously executes a search using the Search api + * Asynchronously executes a search using the Search API * * See Search API on elastic.co */ @@ -365,7 +376,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Executes a search using the Search Scroll api + * Executes a search using the Search Scroll API * * See Search Scroll * API on elastic.co @@ -375,7 +386,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Asynchronously executes a search using the Search Scroll api + * Asynchronously executes a search using the Search Scroll API * * See Search Scroll * API on elastic.co @@ -386,7 +397,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Clears one or more scroll ids using the Clear Scroll api + * Clears one or more scroll ids using the Clear Scroll API * * See * Clear Scroll API on elastic.co @@ -397,7 +408,7 @@ public class RestHighLevelClient implements Closeable { } /** - * Asynchronously clears one or more scroll ids using the Clear Scroll api + * Asynchronously clears one or more scroll ids using the Clear Scroll API * * See * Clear Scroll API on elastic.co diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index b078a983357..e36c445082e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -39,7 +39,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -50,7 +49,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.Collections; @@ -614,14 +612,14 @@ public class CrudIT extends ESRestHighLevelClientTestCase { } }; - ThreadPool threadPool = new ThreadPool(Settings.builder().put("node.name", getClass().getName()).build()); // Pull the client to a variable to work around https://bugs.eclipse.org/bugs/show_bug.cgi?id=514884 RestHighLevelClient hlClient = highLevelClient(); - try(BulkProcessor processor = new BulkProcessor.Builder(hlClient::bulkAsync, listener, threadPool) - .setConcurrentRequests(0) - .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB)) - .setBulkActions(nbItems + 1) - .build()) { + + try (BulkProcessor processor = BulkProcessor.builder(hlClient::bulkAsync, listener) + .setConcurrentRequests(0) + .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB)) + .setBulkActions(nbItems + 1) + .build()) { for (int i = 0; i < nbItems; i++) { String id = String.valueOf(i); boolean erroneous = randomBoolean(); @@ -631,7 +629,7 @@ public class CrudIT extends ESRestHighLevelClientTestCase { if (opType == DocWriteRequest.OpType.DELETE) { if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); } DeleteRequest deleteRequest = new DeleteRequest("index", "test", id); processor.add(deleteRequest); @@ -653,10 +651,10 @@ public class CrudIT extends ESRestHighLevelClientTestCase { } else if (opType == DocWriteRequest.OpType.UPDATE) { UpdateRequest updateRequest = new UpdateRequest("index", "test", id) - .doc(new IndexRequest().source(xContentType, "id", i)); + .doc(new IndexRequest().source(xContentType, "id", i)); if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); } processor.add(updateRequest); } @@ -676,8 +674,6 @@ public class CrudIT extends ESRestHighLevelClientTestCase { assertNull(error.get()); validateBulkResponses(nbItems, errors, bulkResponse, bulkRequest); - - terminate(threadPool); } private void validateBulkResponses(int nbItems, boolean[] errors, BulkResponse bulkResponse, BulkRequest bulkRequest) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java new file mode 100755 index 00000000000..4045e565288 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +public class IndicesClientIT extends ESRestHighLevelClientTestCase { + + public void testDeleteIndex() throws IOException { + { + // Delete index if exists + String indexName = "test_index"; + createIndex(indexName); + + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); + DeleteIndexResponse deleteIndexResponse = + execute(deleteIndexRequest, highLevelClient().indices()::deleteIndex, highLevelClient().indices()::deleteIndexAsync); + assertTrue(deleteIndexResponse.isAcknowledged()); + + assertFalse(indexExists(indexName)); + } + { + // Return 404 if index doesn't exist + String nonExistentIndex = "non_existent_index"; + assertFalse(indexExists(nonExistentIndex)); + + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(nonExistentIndex); + + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> execute(deleteIndexRequest, highLevelClient().indices()::deleteIndex, highLevelClient().indices()::deleteIndexAsync)); + assertEquals(RestStatus.NOT_FOUND, exception.status()); + } + } + + private static void createIndex(String index) throws IOException { + Response response = client().performRequest("PUT", index); + + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + private static boolean indexExists(String index) throws IOException { + Response response = client().performRequest("HEAD", index); + + return response.getStatusLine().getStatusCode() == 200; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java old mode 100644 new mode 100755 index 8f52eb37fe9..3be250d513d --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java @@ -25,6 +25,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -36,6 +37,8 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -44,6 +47,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -74,6 +78,7 @@ import java.util.Map; import java.util.StringJoiner; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import static java.util.Collections.singletonMap; import static org.elasticsearch.client.Request.enforceSameContentType; @@ -139,7 +144,7 @@ public class RequestTests extends ESTestCase { Map expectedParams = new HashMap<>(); - setRandomTimeout(deleteRequest, expectedParams); + setRandomTimeout(deleteRequest::timeout, ReplicationRequest.DEFAULT_TIMEOUT, expectedParams); setRandomRefreshPolicy(deleteRequest, expectedParams); setRandomVersion(deleteRequest, expectedParams); setRandomVersionType(deleteRequest, expectedParams); @@ -240,6 +245,30 @@ public class RequestTests extends ESTestCase { assertEquals(method, request.getMethod()); } + public void testDeleteIndex() throws IOException { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(); + + int numIndices = randomIntBetween(0, 5); + String[] indices = new String[numIndices]; + for (int i = 0; i < numIndices; i++) { + indices[i] = "index-" + randomAlphaOfLengthBetween(2, 5); + } + deleteIndexRequest.indices(indices); + + Map expectedParams = new HashMap<>(); + + setRandomTimeout(deleteIndexRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + setRandomMasterTimeout(deleteIndexRequest, expectedParams); + + setRandomIndicesOptions(deleteIndexRequest::indicesOptions, deleteIndexRequest::indicesOptions, expectedParams); + + Request request = Request.deleteIndex(deleteIndexRequest); + assertEquals("/" + String.join(",", indices), request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertEquals("DELETE", request.getMethod()); + assertNull(request.getEntity()); + } + public void testIndex() throws IOException { String index = randomAlphaOfLengthBetween(3, 10); String type = randomAlphaOfLengthBetween(3, 10); @@ -258,7 +287,7 @@ public class RequestTests extends ESTestCase { } } - setRandomTimeout(indexRequest, expectedParams); + setRandomTimeout(indexRequest::timeout, ReplicationRequest.DEFAULT_TIMEOUT, expectedParams); setRandomRefreshPolicy(indexRequest, expectedParams); // There is some logic around _create endpoint and version/version type @@ -678,20 +707,7 @@ public class RequestTests extends ESTestCase { expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep()); } - if (randomBoolean()) { - searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); - } - expectedParams.put("ignore_unavailable", Boolean.toString(searchRequest.indicesOptions().ignoreUnavailable())); - expectedParams.put("allow_no_indices", Boolean.toString(searchRequest.indicesOptions().allowNoIndices())); - if (searchRequest.indicesOptions().expandWildcardsOpen() && searchRequest.indicesOptions().expandWildcardsClosed()) { - expectedParams.put("expand_wildcards", "open,closed"); - } else if (searchRequest.indicesOptions().expandWildcardsOpen()) { - expectedParams.put("expand_wildcards", "open"); - } else if (searchRequest.indicesOptions().expandWildcardsClosed()) { - expectedParams.put("expand_wildcards", "closed"); - } else { - expectedParams.put("expand_wildcards", "none"); - } + setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams); SearchSourceBuilder searchSourceBuilder = null; if (frequently()) { @@ -903,13 +919,43 @@ public class RequestTests extends ESTestCase { } } - private static void setRandomTimeout(ReplicationRequest request, Map expectedParams) { + private static void setRandomIndicesOptions(Consumer setter, Supplier getter, + Map expectedParams) { + + if (randomBoolean()) { + setter.accept(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), + randomBoolean())); + } + expectedParams.put("ignore_unavailable", Boolean.toString(getter.get().ignoreUnavailable())); + expectedParams.put("allow_no_indices", Boolean.toString(getter.get().allowNoIndices())); + if (getter.get().expandWildcardsOpen() && getter.get().expandWildcardsClosed()) { + expectedParams.put("expand_wildcards", "open,closed"); + } else if (getter.get().expandWildcardsOpen()) { + expectedParams.put("expand_wildcards", "open"); + } else if (getter.get().expandWildcardsClosed()) { + expectedParams.put("expand_wildcards", "closed"); + } else { + expectedParams.put("expand_wildcards", "none"); + } + } + + private static void setRandomTimeout(Consumer setter, TimeValue defaultTimeout, Map expectedParams) { if (randomBoolean()) { String timeout = randomTimeValue(); - request.timeout(timeout); + setter.accept(timeout); expectedParams.put("timeout", timeout); } else { - expectedParams.put("timeout", ReplicationRequest.DEFAULT_TIMEOUT.getStringRep()); + expectedParams.put("timeout", defaultTimeout.getStringRep()); + } + } + + private static void setRandomMasterTimeout(MasterNodeRequest request, Map expectedParams) { + if (randomBoolean()) { + String masterTimeout = randomTimeValue(); + request.masterNodeTimeout(masterTimeout); + expectedParams.put("master_timeout", masterTimeout); + } else { + expectedParams.put("master_timeout", MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT.getStringRep()); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 13b8bf0ea6b..b32351656ca 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -19,13 +19,11 @@ package org.elasticsearch.client.documentation; -import org.elasticsearch.Build; import org.apache.http.HttpEntity; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; @@ -40,7 +38,6 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.main.MainResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationResponse; @@ -49,9 +46,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -64,7 +59,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.Scheduler; import java.io.IOException; import java.util.Collections; @@ -868,31 +863,27 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase { } public void testBulkProcessor() throws InterruptedException, IOException { - Settings settings = Settings.builder().put("node.name", "my-application").build(); RestHighLevelClient client = highLevelClient(); { // tag::bulk-processor-init - ThreadPool threadPool = new ThreadPool(settings); // <1> - - BulkProcessor.Listener listener = new BulkProcessor.Listener() { // <2> + BulkProcessor.Listener listener = new BulkProcessor.Listener() { // <1> @Override public void beforeBulk(long executionId, BulkRequest request) { - // <3> + // <2> } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - // <4> + // <3> } @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - // <5> + // <4> } }; - BulkProcessor bulkProcessor = new BulkProcessor.Builder(client::bulkAsync, listener, threadPool) - .build(); // <6> + BulkProcessor bulkProcessor = BulkProcessor.builder(client::bulkAsync, listener).build(); // <5> // end::bulk-processor-init assertNotNull(bulkProcessor); @@ -917,7 +908,6 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase { // tag::bulk-processor-close bulkProcessor.close(); // end::bulk-processor-close - terminate(threadPool); } { // tag::bulk-processor-listener @@ -944,19 +934,14 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase { }; // end::bulk-processor-listener - ThreadPool threadPool = new ThreadPool(settings); - try { - // tag::bulk-processor-options - BulkProcessor.Builder builder = new BulkProcessor.Builder(client::bulkAsync, listener, threadPool); - builder.setBulkActions(500); // <1> - builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); // <2> - builder.setConcurrentRequests(0); // <3> - builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); // <4> - builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); // <5> - // end::bulk-processor-options - } finally { - terminate(threadPool); - } + // tag::bulk-processor-options + BulkProcessor.Builder builder = BulkProcessor.builder(client::bulkAsync, listener); + builder.setBulkActions(500); // <1> + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); // <2> + builder.setConcurrentRequests(0); // <3> + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); // <4> + builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); // <5> + // end::bulk-processor-options } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java new file mode 100644 index 00000000000..e866fb92aae --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.documentation; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +/** + * This class is used to generate the Java Indices API documentation. + * You need to wrap your code between two tags like: + * // tag::example[] + * // end::example[] + * + * Where example is your tag name. + * + * Then in the documentation, you can extract what is between tag and end tags with + * ["source","java",subs="attributes,callouts,macros"] + * -------------------------------------------------- + * include-tagged::{doc-tests}/CRUDDocumentationIT.java[example] + * -------------------------------------------------- + */ +public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase { + + public void testDeleteIndex() throws IOException { + RestHighLevelClient client = highLevelClient(); + + { + Response createIndexResponse = client().performRequest("PUT", "/posts"); + assertEquals(200, createIndexResponse.getStatusLine().getStatusCode()); + } + + { + // tag::delete-index-request + DeleteIndexRequest request = new DeleteIndexRequest("posts"); // <1> + // end::delete-index-request + + // tag::delete-index-execute + DeleteIndexResponse deleteIndexResponse = client.indices().deleteIndex(request); + // end::delete-index-execute + assertTrue(deleteIndexResponse.isAcknowledged()); + + // tag::delete-index-response + boolean acknowledged = deleteIndexResponse.isAcknowledged(); // <1> + // end::delete-index-response + + // tag::delete-index-execute-async + client.indices().deleteIndexAsync(request, new ActionListener() { + @Override + public void onResponse(DeleteIndexResponse deleteIndexResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }); + // end::delete-index-execute-async + } + + { + DeleteIndexRequest request = new DeleteIndexRequest("posts"); + // tag::delete-index-request-timeout + request.timeout(TimeValue.timeValueMinutes(2)); // <1> + request.timeout("2m"); // <2> + // end::delete-index-request-timeout + // tag::delete-index-request-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.timeout("1m"); // <2> + // end::delete-index-request-masterTimeout + // tag::delete-index-request-indicesOptions + request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::delete-index-request-indicesOptions + } + + { + // tag::delete-index-notfound + try { + DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist"); + DeleteIndexResponse deleteIndexResponse = client.indices().deleteIndex(request); + } catch (ElasticsearchException exception) { + if (exception.status() == RestStatus.NOT_FOUND) { + // <1> + } + } + // end::delete-index-notfound + } + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index da24b5db8aa..e221ed081a5 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -91,8 +92,9 @@ public class RestClient implements Closeable { private static final Log logger = LogFactory.getLog(RestClient.class); private final CloseableHttpAsyncClient client; - //we don't rely on default headers supported by HttpAsyncClient as those cannot be replaced - private final Header[] defaultHeaders; + // We don't rely on default headers supported by HttpAsyncClient as those cannot be replaced. + // These are package private for tests. + final List

defaultHeaders; private final long maxRetryTimeoutMillis; private final String pathPrefix; private final AtomicInteger lastHostIndex = new AtomicInteger(0); @@ -104,7 +106,7 @@ public class RestClient implements Closeable { HttpHost[] hosts, String pathPrefix, FailureListener failureListener) { this.client = client; this.maxRetryTimeoutMillis = maxRetryTimeoutMillis; - this.defaultHeaders = defaultHeaders; + this.defaultHeaders = Collections.unmodifiableList(Arrays.asList(defaultHeaders)); this.failureListener = failureListener; this.pathPrefix = pathPrefix; setHosts(hosts); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 7e4ae6d57ab..750cf609dc6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.Version; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; @@ -36,6 +37,7 @@ import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.jvm.JvmStats; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.monitor.process.ProcessStats; +import org.elasticsearch.node.AdaptiveSelectionStats; import org.elasticsearch.script.ScriptStats; import org.elasticsearch.threadpool.ThreadPoolStats; import org.elasticsearch.transport.TransportStats; @@ -86,6 +88,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private IngestStats ingestStats; + @Nullable + private AdaptiveSelectionStats adaptiveSelectionStats; + NodeStats() { } @@ -95,7 +100,8 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable AllCircuitBreakerStats breaker, @Nullable ScriptStats scriptStats, @Nullable DiscoveryStats discoveryStats, - @Nullable IngestStats ingestStats) { + @Nullable IngestStats ingestStats, + @Nullable AdaptiveSelectionStats adaptiveSelectionStats) { super(node); this.timestamp = timestamp; this.indices = indices; @@ -110,6 +116,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { this.scriptStats = scriptStats; this.discoveryStats = discoveryStats; this.ingestStats = ingestStats; + this.adaptiveSelectionStats = adaptiveSelectionStats; } public long getTimestamp() { @@ -199,6 +206,11 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { return ingestStats; } + @Nullable + public AdaptiveSelectionStats getAdaptiveSelectionStats() { + return adaptiveSelectionStats; + } + public static NodeStats readNodeStats(StreamInput in) throws IOException { NodeStats nodeInfo = new NodeStats(); nodeInfo.readFrom(in); @@ -223,6 +235,11 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { scriptStats = in.readOptionalWriteable(ScriptStats::new); discoveryStats = in.readOptionalWriteable(DiscoveryStats::new); ingestStats = in.readOptionalWriteable(IngestStats::new); + if (in.getVersion().onOrAfter(Version.V_6_1_0)) { + adaptiveSelectionStats = in.readOptionalWriteable(AdaptiveSelectionStats::new); + } else { + adaptiveSelectionStats = null; + } } @Override @@ -246,6 +263,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { out.writeOptionalWriteable(scriptStats); out.writeOptionalWriteable(discoveryStats); out.writeOptionalWriteable(ingestStats); + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + out.writeOptionalWriteable(adaptiveSelectionStats); + } } @Override @@ -306,6 +326,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { if (getIngestStats() != null) { getIngestStats().toXContent(builder, params); } + if (getAdaptiveSelectionStats() != null) { + getAdaptiveSelectionStats().toXContent(builder, params); + } return builder; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index a2098d17361..54ef5b65977 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; @@ -43,6 +44,7 @@ public class NodesStatsRequest extends BaseNodesRequest { private boolean script; private boolean discovery; private boolean ingest; + private boolean adaptiveSelection; public NodesStatsRequest() { } @@ -71,6 +73,7 @@ public class NodesStatsRequest extends BaseNodesRequest { this.script = true; this.discovery = true; this.ingest = true; + this.adaptiveSelection = true; return this; } @@ -90,6 +93,7 @@ public class NodesStatsRequest extends BaseNodesRequest { this.script = false; this.discovery = false; this.ingest = false; + this.adaptiveSelection = false; return this; } @@ -265,6 +269,18 @@ public class NodesStatsRequest extends BaseNodesRequest { return this; } + public boolean adaptiveSelection() { + return adaptiveSelection; + } + + /** + * Should adaptiveSelection statistics be returned. + */ + public NodesStatsRequest adaptiveSelection(boolean adaptiveSelection) { + this.adaptiveSelection = adaptiveSelection; + return this; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -280,6 +296,11 @@ public class NodesStatsRequest extends BaseNodesRequest { script = in.readBoolean(); discovery = in.readBoolean(); ingest = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_6_1_0)) { + adaptiveSelection = in.readBoolean(); + } else { + adaptiveSelection = false; + } } @Override @@ -297,5 +318,8 @@ public class NodesStatsRequest extends BaseNodesRequest { out.writeBoolean(script); out.writeBoolean(discovery); out.writeBoolean(ingest); + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + out.writeBoolean(adaptiveSelection); + } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 56c98ed7db0..f9f3b0826f9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -73,7 +73,7 @@ public class TransportNodesStatsAction extends TransportNodesAction shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index 45fc63de892..2798359d4a8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -176,7 +176,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest PARSER = new ConstructingObjectParser<>("create_index", + true, args -> new CreateIndexResponse((boolean) args[0], (boolean) args[1], (String) args[2])); + + static { + declareAcknowledgedField(PARSER); + PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SHARDS_ACKNOWLEDGED_PARSER, + ObjectParser.ValueType.BOOLEAN); + PARSER.declareField(constructorArg(), (parser, context) -> parser.text(), INDEX_PARSER, ObjectParser.ValueType.STRING); + } private boolean shardsAcked; private String index; @@ -79,7 +102,20 @@ public class CreateIndexResponse extends AcknowledgedResponse { } public void addCustomFields(XContentBuilder builder) throws IOException { - builder.field("shards_acknowledged", isShardsAcked()); - builder.field("index", index()); + builder.field(SHARDS_ACKNOWLEDGED, isShardsAcked()); + builder.field(INDEX, index()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + addAcknowledgedField(builder); + addCustomFields(builder); + builder.endObject(); + return builder; + } + + public static CreateIndexResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.apply(parser, null); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponse.java old mode 100644 new mode 100755 index 509686d3649..8217668e217 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponse.java @@ -22,13 +22,24 @@ package org.elasticsearch.action.admin.indices.delete; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; /** * A response for a delete index action. */ -public class DeleteIndexResponse extends AcknowledgedResponse { +public class DeleteIndexResponse extends AcknowledgedResponse implements ToXContentObject { + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("delete_index", + true, args -> new DeleteIndexResponse((boolean) args[0])); + + static { + declareAcknowledgedField(PARSER); + } DeleteIndexResponse() { } @@ -48,4 +59,16 @@ public class DeleteIndexResponse extends AcknowledgedResponse { super.writeTo(out); writeAcknowledged(out); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + addAcknowledgedField(builder); + builder.endObject(); + return builder; + } + + public static DeleteIndexResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.apply(parser, null); + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 99ad163f48d..1553a528c57 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -104,7 +104,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest> consumer; private final Listener listener; - private final ThreadPool threadPool; - + private final Scheduler scheduler; + private final Runnable onClose; private int concurrentRequests = 1; private int bulkActions = 1000; private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB); private TimeValue flushInterval = null; private BackoffPolicy backoffPolicy = BackoffPolicy.exponentialBackoff(); - /** - * Creates a builder of bulk processor with the client to use and the listener that will be used - * to be notified on the completion of bulk requests. - */ - public Builder(BiConsumer> consumer, Listener listener, ThreadPool threadPool) { + private Builder(BiConsumer> consumer, Listener listener, + Scheduler scheduler, Runnable onClose) { this.consumer = consumer; this.listener = listener; - this.threadPool = threadPool; + this.scheduler = scheduler; + this.onClose = onClose; } /** @@ -155,39 +156,51 @@ public class BulkProcessor implements Closeable { * Builds a new bulk processor. */ public BulkProcessor build() { - return new BulkProcessor(consumer, backoffPolicy, listener, concurrentRequests, bulkActions, bulkSize, flushInterval, threadPool); + return new BulkProcessor(consumer, backoffPolicy, listener, concurrentRequests, bulkActions, bulkSize, flushInterval, + scheduler, onClose); } } public static Builder builder(Client client, Listener listener) { Objects.requireNonNull(client, "client"); Objects.requireNonNull(listener, "listener"); + return new Builder(client::bulk, listener, client.threadPool(), () -> {}); + } - return new Builder(client::bulk, listener, client.threadPool()); + public static Builder builder(BiConsumer> consumer, Listener listener) { + Objects.requireNonNull(consumer, "consumer"); + Objects.requireNonNull(listener, "listener"); + final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = Scheduler.initScheduler(Settings.EMPTY); + return new Builder(consumer, listener, + (delay, executor, command) -> scheduledThreadPoolExecutor.schedule(command, delay.millis(), TimeUnit.MILLISECONDS), + () -> Scheduler.terminate(scheduledThreadPoolExecutor, 10, TimeUnit.SECONDS)); } private final int bulkActions; private final long bulkSize; - private final ThreadPool.Cancellable cancellableFlushTask; + private final Scheduler.Cancellable cancellableFlushTask; private final AtomicLong executionIdGen = new AtomicLong(); private BulkRequest bulkRequest; private final BulkRequestHandler bulkRequestHandler; + private final Scheduler scheduler; + private final Runnable onClose; private volatile boolean closed = false; BulkProcessor(BiConsumer> consumer, BackoffPolicy backoffPolicy, Listener listener, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval, - ThreadPool threadPool) { + Scheduler scheduler, Runnable onClose) { this.bulkActions = bulkActions; this.bulkSize = bulkSize.getBytes(); this.bulkRequest = new BulkRequest(); - this.bulkRequestHandler = new BulkRequestHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests); - + this.scheduler = scheduler; + this.bulkRequestHandler = new BulkRequestHandler(consumer, backoffPolicy, listener, scheduler, concurrentRequests); // Start period flushing task after everything is setup - this.cancellableFlushTask = startFlushTask(flushInterval, threadPool); + this.cancellableFlushTask = startFlushTask(flushInterval, scheduler); + this.onClose = onClose; } /** @@ -200,6 +213,7 @@ public class BulkProcessor implements Closeable { } catch (InterruptedException exc) { Thread.currentThread().interrupt(); } + onClose.run(); } /** @@ -289,9 +303,9 @@ public class BulkProcessor implements Closeable { return this; } - private ThreadPool.Cancellable startFlushTask(TimeValue flushInterval, ThreadPool threadPool) { + private Scheduler.Cancellable startFlushTask(TimeValue flushInterval, Scheduler scheduler) { if (flushInterval == null) { - return new ThreadPool.Cancellable() { + return new Scheduler.Cancellable() { @Override public void cancel() {} @@ -301,9 +315,8 @@ public class BulkProcessor implements Closeable { } }; } - - final Runnable flushRunnable = threadPool.getThreadContext().preserveContext(new Flush()); - return threadPool.scheduleWithFixedDelay(flushRunnable, flushInterval, ThreadPool.Names.GENERIC); + final Runnable flushRunnable = scheduler.preserveContext(new Flush()); + return scheduler.scheduleWithFixedDelay(flushRunnable, flushInterval, ThreadPool.Names.GENERIC); } private void executeIfNeeded() { diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 1b0d89222dd..fbbe6c1bf8a 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -429,6 +429,7 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques if (upsertRequest != null) { upsertRequest.version(version); upsertRequest.versionType(versionType); + upsertRequest.setPipeline(defaultPipeline); } IndexRequest doc = updateRequest.doc(); if (doc != null) { diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java index 52a83b00483..423648bbb71 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java @@ -25,7 +25,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.Scheduler; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; @@ -44,14 +44,13 @@ public final class BulkRequestHandler { private final int concurrentRequests; BulkRequestHandler(BiConsumer> consumer, BackoffPolicy backoffPolicy, - BulkProcessor.Listener listener, ThreadPool threadPool, - int concurrentRequests) { + BulkProcessor.Listener listener, Scheduler scheduler, int concurrentRequests) { assert concurrentRequests >= 0; this.logger = Loggers.getLogger(getClass()); this.consumer = consumer; this.listener = listener; this.concurrentRequests = concurrentRequests; - this.retry = new Retry(EsRejectedExecutionException.class, backoffPolicy, threadPool); + this.retry = new Retry(EsRejectedExecutionException.class, backoffPolicy, scheduler); this.semaphore = new Semaphore(concurrentRequests > 0 ? concurrentRequests : 1); } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java index 8a9ef245f36..9985d23b9ba 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; @@ -41,13 +42,12 @@ import java.util.function.Predicate; public class Retry { private final Class retryOnThrowable; private final BackoffPolicy backoffPolicy; - private final ThreadPool threadPool; + private final Scheduler scheduler; - - public Retry(Class retryOnThrowable, BackoffPolicy backoffPolicy, ThreadPool threadPool) { + public Retry(Class retryOnThrowable, BackoffPolicy backoffPolicy, Scheduler scheduler) { this.retryOnThrowable = retryOnThrowable; this.backoffPolicy = backoffPolicy; - this.threadPool = threadPool; + this.scheduler = scheduler; } /** @@ -58,8 +58,9 @@ public class Retry { * @param listener A listener that is invoked when the bulk request finishes or completes with an exception. The listener is not * @param settings settings */ - public void withBackoff(BiConsumer> consumer, BulkRequest bulkRequest, ActionListener listener, Settings settings) { - RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, listener, settings, threadPool); + public void withBackoff(BiConsumer> consumer, BulkRequest bulkRequest, + ActionListener listener, Settings settings) { + RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, listener, settings, scheduler); r.execute(bulkRequest); } @@ -72,7 +73,8 @@ public class Retry { * @param settings settings * @return a future representing the bulk response returned by the client. */ - public PlainActionFuture withBackoff(BiConsumer> consumer, BulkRequest bulkRequest, Settings settings) { + public PlainActionFuture withBackoff(BiConsumer> consumer, + BulkRequest bulkRequest, Settings settings) { PlainActionFuture future = PlainActionFuture.newFuture(); withBackoff(consumer, bulkRequest, future, settings); return future; @@ -80,7 +82,7 @@ public class Retry { static class RetryHandler implements ActionListener { private final Logger logger; - private final ThreadPool threadPool; + private final Scheduler scheduler; private final BiConsumer> consumer; private final ActionListener listener; private final Iterator backoff; @@ -95,13 +97,13 @@ public class Retry { RetryHandler(Class retryOnThrowable, BackoffPolicy backoffPolicy, BiConsumer> consumer, ActionListener listener, - Settings settings, ThreadPool threadPool) { + Settings settings, Scheduler scheduler) { this.retryOnThrowable = retryOnThrowable; this.backoff = backoffPolicy.iterator(); this.consumer = consumer; this.listener = listener; this.logger = Loggers.getLogger(getClass(), settings); - this.threadPool = threadPool; + this.scheduler = scheduler; // in contrast to System.currentTimeMillis(), nanoTime() uses a monotonic clock under the hood this.startTimestampNanos = System.nanoTime(); } @@ -136,8 +138,8 @@ public class Retry { assert backoff.hasNext(); TimeValue next = backoff.next(); logger.trace("Retry of bulk request scheduled in {} ms.", next.millis()); - Runnable command = threadPool.getThreadContext().preserveContext(() -> this.execute(bulkRequestForRetry)); - scheduledRequestFuture = threadPool.schedule(next, ThreadPool.Names.SAME, command); + Runnable command = scheduler.preserveContext(() -> this.execute(bulkRequestForRetry)); + scheduledRequestFuture = scheduler.schedule(next, ThreadPool.Names.SAME, command); } private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) { diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index c7f1fa5dc5c..1c25cd7ac37 100644 --- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -77,7 +77,7 @@ abstract class AbstractSearchAsyncAction exten ActionListener listener, GroupShardsIterator shardsIts, TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, SearchPhaseResults resultConsumer, int maxConcurrentShardRequests) { - super(name, request, shardsIts, logger, maxConcurrentShardRequests); + super(name, request, shardsIts, logger, maxConcurrentShardRequests, executor); this.timeProvider = timeProvider; this.logger = logger; this.searchTransportService = searchTransportService; diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index bc673644a06..53ce4299c54 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -88,10 +88,9 @@ final class ExpandSearchPhase extends SearchPhase { } for (InnerHitBuilder innerHitBuilder : innerHitBuilders) { SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder) - .query(groupQuery); - SearchRequest groupRequest = new SearchRequest(searchRequest.indices()) - .types(searchRequest.types()) - .source(sourceBuilder); + .query(groupQuery) + .postFilter(searchRequest.source().postFilter()); + SearchRequest groupRequest = buildExpandSearchRequest(searchRequest, sourceBuilder); multiRequest.add(groupRequest); } } @@ -120,6 +119,21 @@ final class ExpandSearchPhase extends SearchPhase { } } + private SearchRequest buildExpandSearchRequest(SearchRequest orig, SearchSourceBuilder sourceBuilder) { + SearchRequest groupRequest = new SearchRequest(orig.indices()) + .types(orig.types()) + .source(sourceBuilder) + .indicesOptions(orig.indicesOptions()) + .requestCache(orig.requestCache()) + .preference(orig.preference()) + .routing(orig.routing()) + .searchType(orig.searchType()); + if (orig.isMaxConcurrentShardRequestsSet()) { + groupRequest.setMaxConcurrentShardRequests(orig.getMaxConcurrentShardRequests()); + } + return groupRequest; + } + private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options) { SearchSourceBuilder groupSource = new SearchSourceBuilder(); groupSource.from(options.getFrom()); diff --git a/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java index a68d1d599c5..0da74242937 100644 --- a/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java +++ b/core/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java @@ -26,12 +26,15 @@ import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.transport.ConnectTransportException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -45,18 +48,30 @@ import java.util.stream.Stream; */ abstract class InitialSearchPhase extends SearchPhase { private final SearchRequest request; + private final GroupShardsIterator toSkipShardsIts; private final GroupShardsIterator shardsIts; private final Logger logger; private final int expectedTotalOps; private final AtomicInteger totalOps = new AtomicInteger(); private final AtomicInteger shardExecutionIndex = new AtomicInteger(0); private final int maxConcurrentShardRequests; + private final Executor executor; InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator shardsIts, Logger logger, - int maxConcurrentShardRequests) { + int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; - this.shardsIts = shardsIts; + final List toSkipIterators = new ArrayList<>(); + final List iterators = new ArrayList<>(); + for (final SearchShardIterator iterator : shardsIts) { + if (iterator.skip()) { + toSkipIterators.add(iterator); + } else { + iterators.add(iterator); + } + } + this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); + this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. @@ -64,6 +79,7 @@ abstract class InitialSearchPhase extends // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); + this.executor = executor; } private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId, @@ -71,19 +87,19 @@ abstract class InitialSearchPhase extends // we always add the shard failure for a specific shard instance // we do make sure to clean it on a successful response from a shard SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId(), shardIt.getClusterAlias(), - shardIt.getOriginalIndices()); + shardIt.getOriginalIndices()); onShardFailure(shardIndex, shardTarget, e); if (totalOps.incrementAndGet() == expectedTotalOps) { if (logger.isDebugEnabled()) { if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( - (Supplier) () -> new ParameterizedMessage( - "{}: Failed to execute [{}]", - shard != null ? shard.shortSummary() : - shardIt.shardId(), - request), - e); + (Supplier) () -> new ParameterizedMessage( + "{}: Failed to execute [{}]", + shard != null ? shard.shortSummary() : + shardIt.shardId(), + request), + e); } else if (logger.isTraceEnabled()) { logger.trace((Supplier) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e); } @@ -94,32 +110,27 @@ abstract class InitialSearchPhase extends final boolean lastShard = nextShard == null; // trace log this exception logger.trace( - (Supplier) () -> new ParameterizedMessage( - "{}: Failed to execute [{}] lastShard [{}]", - shard != null ? shard.shortSummary() : shardIt.shardId(), - request, - lastShard), - e); + (Supplier) () -> new ParameterizedMessage( + "{}: Failed to execute [{}] lastShard [{}]", + shard != null ? shard.shortSummary() : shardIt.shardId(), + request, + lastShard), + e); if (!lastShard) { - try { - performPhaseOnShard(shardIndex, shardIt, nextShard); - } catch (Exception inner) { - inner.addSuppressed(e); - onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, inner); - } + performPhaseOnShard(shardIndex, shardIt, nextShard); } else { maybeExecuteNext(); // move to the next execution if needed // no more shards active, add a failure if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( - (Supplier) () -> new ParameterizedMessage( - "{}: Failed to execute [{}] lastShard [{}]", - shard != null ? shard.shortSummary() : - shardIt.shardId(), - request, - lastShard), - e); + (Supplier) () -> new ParameterizedMessage( + "{}: Failed to execute [{}] lastShard [{}]", + shard != null ? shard.shortSummary() : + shardIt.shardId(), + request, + lastShard), + e); } } } @@ -128,14 +139,18 @@ abstract class InitialSearchPhase extends @Override public final void run() throws IOException { - boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests); - assert success; - for (int i = 0; i < maxConcurrentShardRequests; i++) { - SearchShardIterator shardRoutings = shardsIts.get(i); - if (shardRoutings.skip()) { - skipShard(shardRoutings); - } else { - performPhaseOnShard(i, shardRoutings, shardRoutings.nextOrNull()); + for (final SearchShardIterator iterator : toSkipShardsIts) { + assert iterator.skip(); + skipShard(iterator); + } + if (shardsIts.size() > 0) { + int maxConcurrentShardRequests = Math.min(this.maxConcurrentShardRequests, shardsIts.size()); + final boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests); + assert success; + for (int index = 0; index < maxConcurrentShardRequests; index++) { + final SearchShardIterator shardRoutings = shardsIts.get(index); + assert shardRoutings.skip() == false; + performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); } } } @@ -143,38 +158,71 @@ abstract class InitialSearchPhase extends private void maybeExecuteNext() { final int index = shardExecutionIndex.getAndIncrement(); if (index < shardsIts.size()) { - SearchShardIterator shardRoutings = shardsIts.get(index); - if (shardRoutings.skip()) { - skipShard(shardRoutings); - } else { - performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); - } + final SearchShardIterator shardRoutings = shardsIts.get(index); + performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); } } + private void maybeFork(final Thread thread, final Runnable runnable) { + if (thread == Thread.currentThread()) { + fork(runnable); + } else { + runnable.run(); + } + } + + private void fork(final Runnable runnable) { + executor.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + + } + + @Override + protected void doRun() throws Exception { + runnable.run(); + } + + @Override + public boolean isForceExecution() { + // we can not allow a stuffed queue to reject execution here + return true; + } + }); + } + private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final ShardRouting shard) { + /* + * We capture the thread that this phase is starting on. When we are called back after executing the phase, we are either on the + * same thread (because we never went async, or the same thread was selected from the thread pool) or a different thread. If we + * continue on the same thread in the case that we never went async and this happens repeatedly we will end up recursing deeply and + * could stack overflow. To prevent this, we fork if we are called back on the same thread that execution started on and otherwise + * we can continue (cf. InitialSearchPhase#maybeFork). + */ + final Thread thread = Thread.currentThread(); if (shard == null) { - onShardFailure(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId())); + fork(() -> onShardFailure(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()))); } else { try { executePhaseOnShard(shardIt, shard, new SearchActionListener(new SearchShardTarget(shard.currentNodeId(), shardIt.shardId(), shardIt.getClusterAlias(), shardIt.getOriginalIndices()), shardIndex) { @Override public void innerOnResponse(FirstResult result) { - onShardResult(result, shardIt); + maybeFork(thread, () -> onShardResult(result, shardIt)); } @Override public void onFailure(Exception t) { - onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, t); + maybeFork(thread, () -> onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, t)); } }); - } catch (ConnectTransportException | IllegalArgumentException ex) { - // we are getting the connection early here so we might run into nodes that are not connected. in that case we move on to - // the next shard. previously when using discovery nodes here we had a special case for null when a node was not connected - // at all which is not not needed anymore. - onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, ex); + } catch (final Exception e) { + /* + * It is possible to run into connection exceptions here because we are getting the connection early and might run in to + * nodes that are not connected. In this case, on shard failure will move us to the next shard copy. + */ + fork(() -> onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, e)); } } } @@ -204,7 +252,7 @@ abstract class InitialSearchPhase extends } else if (xTotalOps > expectedTotalOps) { throw new AssertionError("unexpected higher total ops [" + xTotalOps + "] compared to expected [" + expectedTotalOps + "]"); - } else { + } else if (shardsIt.skip() == false) { maybeExecuteNext(); } } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 8a4c8b0882f..509f89a7542 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -60,6 +60,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.function.BiFunction; @@ -94,6 +95,10 @@ public class SearchTransportService extends AbstractComponent { this.responseWrapper = responseWrapper; } + public Map getClientConnections() { + return Collections.unmodifiableMap(clientConnections); + } + public void sendFreeContext(Transport.Connection connection, final long contextId, OriginalIndices originalIndices) { transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(originalIndices, contextId), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener() { diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 7eb939ca827..f2ba62fefd4 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -131,7 +131,8 @@ public class ShardSearchFailure implements ShardOperationFailedException { @Override public String toString() { - return "shard [" + (shardTarget == null ? "_na" : shardTarget) + "], reason [" + reason + "], cause [" + (cause == null ? "_na" : ExceptionsHelper.stackTrace(cause)) + "]"; + return "shard [" + (shardTarget == null ? "_na" : shardTarget) + "], reason [" + reason + "], cause [" + + (cause == null ? "_na" : ExceptionsHelper.stackTrace(cause)) + "]"; } public static ShardSearchFailure readShardSearchFailure(StreamInput in) throws IOException { @@ -210,9 +211,12 @@ public class ShardSearchFailure implements ShardOperationFailedException { parser.skipChildren(); } } - return new ShardSearchFailure(exception, - new SearchShardTarget(nodeId, - new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), null, OriginalIndices.NONE)); + SearchShardTarget searchShardTarget = null; + if (nodeId != null) { + searchShardTarget = new SearchShardTarget(nodeId, + new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), null, OriginalIndices.NONE); + } + return new ShardSearchFailure(exception, searchShardTarget); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java old mode 100644 new mode 100755 index cdac96a7a79..e4467964722 --- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java +++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java @@ -19,17 +19,32 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + /** * Abstract class that allows to mark action responses that support acknowledgements. * Facilitates consistency across different api. */ public abstract class AcknowledgedResponse extends ActionResponse { + private static final String ACKNOWLEDGED = "acknowledged"; + private static final ParseField ACKNOWLEDGED_PARSER = new ParseField(ACKNOWLEDGED); + + protected static void declareAcknowledgedField(ConstructingObjectParser PARSER) { + PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), ACKNOWLEDGED_PARSER, + ObjectParser.ValueType.BOOLEAN); + } + private boolean acknowledged; protected AcknowledgedResponse() { @@ -61,4 +76,8 @@ public abstract class AcknowledgedResponse extends ActionResponse { protected void writeAcknowledged(StreamOutput out) throws IOException { out.writeBoolean(acknowledged); } + + protected void addAcknowledgedField(XContentBuilder builder) throws IOException { + builder.field(ACKNOWLEDGED, isAcknowledged()); + } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java index f1616ba0eea..0b9913f7f06 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -21,6 +21,7 @@ package org.elasticsearch.bootstrap; import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginInfo; @@ -73,6 +74,9 @@ final class Spawner implements Closeable { */ try (DirectoryStream stream = Files.newDirectoryStream(pluginsFile)) { for (final Path plugin : stream) { + if (FileSystemUtils.isDesktopServicesStore(plugin)) { + continue; + } final PluginInfo info = PluginInfo.readFromProperties(plugin); final Path spawnPath = Platforms.nativeControllerPath(plugin); if (!Files.isRegularFile(spawnPath)) { diff --git a/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java index ecb313780f6..73814a4311a 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java @@ -199,7 +199,6 @@ final class SystemCallFilter { static final int SECCOMP_RET_ALLOW = 0x7FFF0000; // some errno constants for error checking/handling - static final int EPERM = 0x01; static final int EACCES = 0x0D; static final int EFAULT = 0x0E; static final int EINVAL = 0x16; @@ -272,27 +271,6 @@ final class SystemCallFilter { "with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in"); } - // pure paranoia: - - // check that unimplemented syscalls actually return ENOSYS - // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795) - if (linux_syscall(999) >= 0) { - throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade"); - } - - switch (Native.getLastError()) { - case ENOSYS: - break; // ok - case EPERM: - // NOT ok, but likely a docker container - if (logger.isDebugEnabled()) { - logger.debug("syscall(BOGUS) bogusly gets EPERM instead of ENOSYS"); - } - break; - default: - throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade"); - } - // try to check system calls really are who they claim // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57) final int bogusArg = 0xf7a46a5c; diff --git a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index b2c6340ebe4..a976fe779db 100644 --- a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.io; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; @@ -65,6 +66,16 @@ public final class FileSystemUtils { return fileName.toString().startsWith("."); } + /** + * Check whether the file denoted by the given path is a desktop services store created by Finder on macOS. + * + * @param path the path + * @return true if the current system is macOS and the specified file appears to be a desktop services store file + */ + public static boolean isDesktopServicesStore(final Path path) { + return Constants.MAC_OS_X && Files.isRegularFile(path) && ".DS_Store".equals(path.getFileName().toString()); + } + /** * Appends the path to the given base and strips N elements off the path if strip is > 0. */ diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/TimedRunnable.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/TimedRunnable.java index 2ee80badb74..2d8934ba3b3 100644 --- a/core/src/main/java/org/elasticsearch/common/util/concurrent/TimedRunnable.java +++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/TimedRunnable.java @@ -23,19 +23,19 @@ package org.elasticsearch.common.util.concurrent; * A class used to wrap a {@code Runnable} that allows capturing the time of the task since creation * through execution as well as only execution time. */ -class TimedRunnable implements Runnable { +class TimedRunnable extends AbstractRunnable { private final Runnable original; private final long creationTimeNanos; private long startTimeNanos; private long finishTimeNanos = -1; - TimedRunnable(Runnable original) { + TimedRunnable(final Runnable original) { this.original = original; this.creationTimeNanos = System.nanoTime(); } @Override - public void run() { + public void doRun() { try { startTimeNanos = System.nanoTime(); original.run(); @@ -44,6 +44,32 @@ class TimedRunnable implements Runnable { } } + @Override + public void onRejection(final Exception e) { + if (original instanceof AbstractRunnable) { + ((AbstractRunnable) original).onRejection(e); + } + } + + @Override + public void onAfter() { + if (original instanceof AbstractRunnable) { + ((AbstractRunnable) original).onAfter(); + } + } + + @Override + public void onFailure(final Exception e) { + if (original instanceof AbstractRunnable) { + ((AbstractRunnable) original).onFailure(e); + } + } + + @Override + public boolean isForceExecution() { + return original instanceof AbstractRunnable && ((AbstractRunnable) original).isForceExecution(); + } + /** * Return the time since this task was created until it finished running. * If the task is still running or has not yet been run, returns -1. @@ -67,4 +93,5 @@ class TimedRunnable implements Runnable { } return finishTimeNanos - startTimeNanos; } + } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java index d77abac1980..03f6b14f525 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java @@ -453,7 +453,7 @@ public final class ConstructingObjectParser extends AbstractObje * use of ConstructingObjectParser. You should be using ObjectParser instead. Since this is more of a programmer error and the * parser ought to still work we just assert this. */ - assert false == constructorArgInfos.isEmpty() : "[" + objectParser.getName() + "] must configure at least on constructor " + assert false == constructorArgInfos.isEmpty() : "[" + objectParser.getName() + "] must configure at least one constructor " + "argument. If it doesn't have any it should use ObjectParser instead of ConstructingObjectParser. This is a bug " + "in the parser declaration."; // All missing constructor arguments were optional. Just build the target and return it. diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java index 37e916d65fb..a4c788d5c22 100644 --- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java +++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java @@ -19,6 +19,7 @@ package org.elasticsearch.discovery; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -26,33 +27,48 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.discovery.zen.PendingClusterStateStats; +import org.elasticsearch.discovery.zen.PublishClusterStateStats; import java.io.IOException; public class DiscoveryStats implements Writeable, ToXContentFragment { - @Nullable private final PendingClusterStateStats queueStats; + private final PublishClusterStateStats publishStats; - public DiscoveryStats(PendingClusterStateStats queueStats) { + public DiscoveryStats(PendingClusterStateStats queueStats, PublishClusterStateStats publishStats) { this.queueStats = queueStats; + this.publishStats = publishStats; } public DiscoveryStats(StreamInput in) throws IOException { queueStats = in.readOptionalWriteable(PendingClusterStateStats::new); + + if (in.getVersion().onOrAfter(Version.V_6_1_0)) { + publishStats = in.readOptionalWriteable(PublishClusterStateStats::new); + } else { + publishStats = null; + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(queueStats); + + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + out.writeOptionalWriteable(publishStats); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.DISCOVERY); - if (queueStats != null ){ + if (queueStats != null) { queueStats.toXContent(builder, params); } + if (publishStats != null) { + publishStats.toXContent(builder, params); + } builder.endObject(); return builder; } @@ -64,4 +80,8 @@ public class DiscoveryStats implements Writeable, ToXContentFragment { public PendingClusterStateStats getQueueStats() { return queueStats; } + + public PublishClusterStateStats getPublishStats() { + return publishStats; + } } diff --git a/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java index 2f3124010cc..2a32caabc77 100644 --- a/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java @@ -21,7 +21,6 @@ package org.elasticsearch.discovery.single; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -34,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.discovery.zen.PendingClusterStateStats; +import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -94,7 +94,7 @@ public class SingleNodeDiscovery extends AbstractLifecycleComponent implements D @Override public DiscoveryStats stats() { - return new DiscoveryStats((PendingClusterStateStats) null); + return new DiscoveryStats(null, null); } @Override diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java index ae469d162ae..95de654928e 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java @@ -65,6 +65,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; public class PublishClusterStateAction extends AbstractComponent { @@ -90,6 +91,10 @@ public class PublishClusterStateAction extends AbstractComponent { private final IncomingClusterStateListener incomingClusterStateListener; private final DiscoverySettings discoverySettings; + private final AtomicLong fullClusterStateReceivedCount = new AtomicLong(); + private final AtomicLong incompatibleClusterStateDiffReceivedCount = new AtomicLong(); + private final AtomicLong compatibleClusterStateDiffReceivedCount = new AtomicLong(); + public PublishClusterStateAction( Settings settings, TransportService transportService, @@ -380,11 +385,13 @@ public class PublishClusterStateAction extends AbstractComponent { // If true we received full cluster state - otherwise diffs if (in.readBoolean()) { incomingState = ClusterState.readFrom(in, transportService.getLocalNode()); + fullClusterStateReceivedCount.incrementAndGet(); logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length()); } else if (lastSeenClusterState != null) { Diff diff = ClusterState.readDiffFrom(in, lastSeenClusterState.nodes().getLocalNode()); incomingState = diff.apply(lastSeenClusterState); + compatibleClusterStateDiffReceivedCount.incrementAndGet(); logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", incomingState.version(), incomingState.stateUUID(), request.bytes().length()); } else { @@ -394,6 +401,9 @@ public class PublishClusterStateAction extends AbstractComponent { incomingClusterStateListener.onIncomingClusterState(incomingState); lastSeenClusterState = incomingState; } + } catch (IncompatibleClusterStateVersionException e) { + incompatibleClusterStateDiffReceivedCount.incrementAndGet(); + throw e; } finally { IOUtils.close(in); } @@ -636,4 +646,11 @@ public class PublishClusterStateAction extends AbstractComponent { publishingTimedOut.set(isTimedOut); } } + + public PublishClusterStateStats stats() { + return new PublishClusterStateStats( + fullClusterStateReceivedCount.get(), + incompatibleClusterStateDiffReceivedCount.get(), + compatibleClusterStateDiffReceivedCount.get()); + } } diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateStats.java b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateStats.java new file mode 100644 index 00000000000..8a848198759 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateStats.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.zen; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Class encapsulating stats about the PublishClusterStateAction + */ +public class PublishClusterStateStats implements Writeable, ToXContentObject { + + private final long fullClusterStateReceivedCount; + private final long incompatibleClusterStateDiffReceivedCount; + private final long compatibleClusterStateDiffReceivedCount; + + /** + * @param fullClusterStateReceivedCount the number of times this node has received a full copy of the cluster state from the master. + * @param incompatibleClusterStateDiffReceivedCount the number of times this node has received a cluster-state diff from the master. + * @param compatibleClusterStateDiffReceivedCount the number of times that received cluster-state diffs were compatible with + */ + public PublishClusterStateStats(long fullClusterStateReceivedCount, + long incompatibleClusterStateDiffReceivedCount, + long compatibleClusterStateDiffReceivedCount) { + this.fullClusterStateReceivedCount = fullClusterStateReceivedCount; + this.incompatibleClusterStateDiffReceivedCount = incompatibleClusterStateDiffReceivedCount; + this.compatibleClusterStateDiffReceivedCount = compatibleClusterStateDiffReceivedCount; + } + + public PublishClusterStateStats(StreamInput in) throws IOException { + fullClusterStateReceivedCount = in.readVLong(); + incompatibleClusterStateDiffReceivedCount = in.readVLong(); + compatibleClusterStateDiffReceivedCount = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(fullClusterStateReceivedCount); + out.writeVLong(incompatibleClusterStateDiffReceivedCount); + out.writeVLong(compatibleClusterStateDiffReceivedCount); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("published_cluster_states"); + { + builder.field("full_states", fullClusterStateReceivedCount); + builder.field("incompatible_diffs", incompatibleClusterStateDiffReceivedCount); + builder.field("compatible_diffs", compatibleClusterStateDiffReceivedCount); + } + builder.endObject(); + return builder; + } + + long getFullClusterStateReceivedCount() { return fullClusterStateReceivedCount; } + + long getIncompatibleClusterStateDiffReceivedCount() { return incompatibleClusterStateDiffReceivedCount; } + + long getCompatibleClusterStateDiffReceivedCount() { return compatibleClusterStateDiffReceivedCount; } + + @Override + public String toString() { + return "PublishClusterStateStats(full=" + fullClusterStateReceivedCount + + ", incompatible=" + incompatibleClusterStateDiffReceivedCount + + ", compatible=" + compatibleClusterStateDiffReceivedCount + + ")"; + } +} diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index ba5d5213e17..d688a5d5cdb 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -412,8 +412,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover @Override public DiscoveryStats stats() { - PendingClusterStateStats queueStats = pendingStatesQueue.stats(); - return new DiscoveryStats(queueStats); + return new DiscoveryStats(pendingStatesQueue.stats(), publishClusterState.stats()); } public DiscoverySettings getDiscoverySettings() { diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index ecebe411534..9c583351174 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -845,6 +845,29 @@ public final class NodeEnvironment implements Closeable { return shardIds; } + /** + * Find all the shards for this index, returning a map of the {@code NodePath} to the number of shards on that path + * @param index the index by which to filter shards + * @return a map of NodePath to count of the shards for the index on that path + * @throws IOException if an IOException occurs + */ + public Map shardCountPerPath(final Index index) throws IOException { + assert index != null; + if (nodePaths == null || locks == null) { + throw new IllegalStateException("node is not configured to store local location"); + } + assertEnvIsLocked(); + final Map shardCountPerPath = new HashMap<>(); + final String indexUniquePathId = index.getUUID(); + for (final NodePath nodePath : nodePaths) { + Path indexLocation = nodePath.indicesPath.resolve(indexUniquePathId); + if (Files.isDirectory(indexLocation)) { + shardCountPerPath.put(nodePath, (long) findAllShardsForIndex(indexLocation, index).size()); + } + } + return shardCountPerPath; + } + private static Set findAllShardsForIndex(Path indexPath, Index index) throws IOException { assert indexPath.getFileName().toString().equals(index.getUUID()); Set shardIds = new HashSet<>(); diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index c01dd986e25..1cf18cb4ee5 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -708,12 +708,11 @@ public abstract class Engine implements Closeable { protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) { ensureOpen(); Map segments = new HashMap<>(); - // first, go over and compute the search ones... - Searcher searcher = acquireSearcher("segments"); - try { + try (Searcher searcher = acquireSearcher("segments")){ for (LeafReaderContext reader : searcher.reader().leaves()) { - SegmentCommitInfo info = segmentReader(reader.reader()).getSegmentInfo(); + final SegmentReader segmentReader = segmentReader(reader.reader()); + SegmentCommitInfo info = segmentReader.getSegmentInfo(); assert !segments.containsKey(info.info.name); Segment segment = new Segment(info.info.name); segment.search = true; @@ -726,7 +725,6 @@ public abstract class Engine implements Closeable { } catch (IOException e) { logger.trace((Supplier) () -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); } - final SegmentReader segmentReader = segmentReader(reader.reader()); segment.memoryInBytes = segmentReader.ramBytesUsed(); segment.segmentSort = info.info.getIndexSort(); if (verbose) { @@ -736,8 +734,6 @@ public abstract class Engine implements Closeable { // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); } - } finally { - searcher.close(); } // now, correlate or add the committed ones... diff --git a/core/src/main/java/org/elasticsearch/index/get/GetResult.java b/core/src/main/java/org/elasticsearch/index/get/GetResult.java index a47bb8be89e..75e283b4191 100644 --- a/core/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/core/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -82,7 +82,7 @@ public class GetResult implements Streamable, Iterable, ToXConten } /** - * Does the document exists. + * Does the document exist. */ public boolean isExists() { return exists; diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java index c67cceec0d1..61029f70e8f 100644 --- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java @@ -43,6 +43,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import static org.elasticsearch.common.lucene.search.Queries.newLenientFieldQuery; + public class MultiMatchQuery extends MatchQuery { private Float groupTieBreaker = null; @@ -204,7 +206,7 @@ public class MultiMatchQuery extends MatchQuery { for (int i = 0; i < terms.length; i++) { values[i] = terms[i].bytes(); } - return MultiMatchQuery.blendTerms(context, values, commonTermsCutoff, tieBreaker, blendedFields); + return MultiMatchQuery.blendTerms(context, values, commonTermsCutoff, tieBreaker, lenient, blendedFields); } @Override @@ -212,7 +214,7 @@ public class MultiMatchQuery extends MatchQuery { if (blendedFields == null) { return super.blendTerm(term, fieldType); } - return MultiMatchQuery.blendTerm(context, term.bytes(), commonTermsCutoff, tieBreaker, blendedFields); + return MultiMatchQuery.blendTerm(context, term.bytes(), commonTermsCutoff, tieBreaker, lenient, blendedFields); } @Override @@ -227,12 +229,12 @@ public class MultiMatchQuery extends MatchQuery { } static Query blendTerm(QueryShardContext context, BytesRef value, Float commonTermsCutoff, float tieBreaker, - FieldAndFieldType... blendedFields) { - return blendTerms(context, new BytesRef[] {value}, commonTermsCutoff, tieBreaker, blendedFields); + boolean lenient, FieldAndFieldType... blendedFields) { + return blendTerms(context, new BytesRef[] {value}, commonTermsCutoff, tieBreaker, lenient, blendedFields); } static Query blendTerms(QueryShardContext context, BytesRef[] values, Float commonTermsCutoff, float tieBreaker, - FieldAndFieldType... blendedFields) { + boolean lenient, FieldAndFieldType... blendedFields) { List queries = new ArrayList<>(); Term[] terms = new Term[blendedFields.length * values.length]; float[] blendedBoost = new float[blendedFields.length * values.length]; @@ -242,19 +244,12 @@ public class MultiMatchQuery extends MatchQuery { Query query; try { query = ft.fieldType.termQuery(term, context); - } catch (IllegalArgumentException e) { - // the query expects a certain class of values such as numbers - // of ip addresses and the value can't be parsed, so ignore this - // field - continue; - } catch (ElasticsearchParseException parseException) { - // date fields throw an ElasticsearchParseException with the - // underlying IAE as the cause, ignore this field if that is - // the case - if (parseException.getCause() instanceof IllegalArgumentException) { - continue; + } catch (RuntimeException e) { + if (lenient) { + query = newLenientFieldQuery(ft.fieldType.name(), e); + } else { + throw e; } - throw parseException; } float boost = ft.boost; while (query instanceof BoostQuery) { diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java index 37c7b13ec79..63697702910 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java @@ -180,12 +180,19 @@ public final class ShardSearchStats implements SearchOperationListener { public void onFreeScrollContext(SearchContext context) { totalStats.scrollCurrent.dec(); assert totalStats.scrollCurrent.count() >= 0; - totalStats.scrollMetric.inc(System.nanoTime() - context.getOriginNanoTime()); + totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - context.getOriginNanoTime())); } static final class StatsHolder { public final MeanMetric queryMetric = new MeanMetric(); public final MeanMetric fetchMetric = new MeanMetric(); + /* We store scroll statistics in microseconds because with nanoseconds we run the risk of overflowing the total stats if there are + * many scrolls. For example, on a system with 2^24 scrolls that have been executed, each executing for 2^10 seconds, then using + * nanoseconds would require a numeric representation that can represent at least 2^24 * 2^10 * 10^9 > 2^24 * 2^10 * 2^29 = 2^63 + * which exceeds the largest value that can be represented by a long. By using microseconds, we enable capturing one-thousand + * times as many scrolls (i.e., billions of scrolls which at one per second would take 32 years to occur), or scrolls that execute + * for one-thousand times as long (i.e., scrolls that execute for almost twelve days on average). + */ public final MeanMetric scrollMetric = new MeanMetric(); public final MeanMetric suggestMetric = new MeanMetric(); public final CounterMetric queryCurrent = new CounterMetric(); @@ -197,7 +204,7 @@ public final class ShardSearchStats implements SearchOperationListener { return new SearchStats.Stats( queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(), - scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(), + scrollMetric.count(), TimeUnit.MICROSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(), suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), suggestCurrent.count() ); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java b/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java index 6d4f5ee7815..ffad0f085f7 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java +++ b/core/src/main/java/org/elasticsearch/index/shard/DocsStats.java @@ -19,11 +19,13 @@ package org.elasticsearch.index.shard; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.store.StoreStats; import java.io.IOException; @@ -31,22 +33,25 @@ public class DocsStats implements Streamable, ToXContentFragment { long count = 0; long deleted = 0; + long totalSizeInBytes = 0; public DocsStats() { } - public DocsStats(long count, long deleted) { + public DocsStats(long count, long deleted, long totalSizeInBytes) { this.count = count; this.deleted = deleted; + this.totalSizeInBytes = totalSizeInBytes; } - public void add(DocsStats docsStats) { - if (docsStats == null) { + public void add(DocsStats other) { + if (other == null) { return; } - count += docsStats.count; - deleted += docsStats.deleted; + this.totalSizeInBytes += other.totalSizeInBytes; + this.count += other.count; + this.deleted += other.deleted; } public long getCount() { @@ -57,16 +62,40 @@ public class DocsStats implements Streamable, ToXContentFragment { return this.deleted; } + /** + * Returns the total size in bytes of all documents in this stats. + * This value may be more reliable than {@link StoreStats#getSizeInBytes()} in estimating the index size. + */ + public long getTotalSizeInBytes() { + return totalSizeInBytes; + } + + /** + * Returns the average size in bytes of all documents in this stats. + */ + public long getAverageSizeInBytes() { + long totalDocs = count + deleted; + return totalDocs == 0 ? 0 : totalSizeInBytes / totalDocs; + } + @Override public void readFrom(StreamInput in) throws IOException { count = in.readVLong(); deleted = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_6_1_0)) { + totalSizeInBytes = in.readVLong(); + } else { + totalSizeInBytes = -1; + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(count); out.writeVLong(deleted); + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + out.writeVLong(totalSizeInBytes); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 09fa2a264ef..c6d6e1c5aa6 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -879,9 +879,18 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } public DocsStats docStats() { - try (Engine.Searcher searcher = acquireSearcher("doc_stats")) { - return new DocsStats(searcher.reader().numDocs(), searcher.reader().numDeletedDocs()); + long numDocs = 0; + long numDeletedDocs = 0; + long sizeInBytes = 0; + List segments = segments(false); + for (Segment segment : segments) { + if (segment.search) { + numDocs += segment.getNumDocs(); + numDeletedDocs += segment.getDeletedDocs(); + sizeInBytes += segment.getSizeInBytes(); + } } + return new DocsStats(numDocs, numDeletedDocs, sizeInBytes); } /** diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java index eb2d530fd99..4a18bcdd5c7 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java @@ -31,7 +31,11 @@ import java.math.BigInteger; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public final class ShardPath { public static final String INDEX_FOLDER_NAME = "index"; @@ -189,23 +193,49 @@ public final class ShardPath { // TODO - do we need something more extensible? Yet, this does the job for now... final NodeEnvironment.NodePath[] paths = env.nodePaths(); - NodeEnvironment.NodePath bestPath = null; - BigInteger maxUsableBytes = BigInteger.valueOf(Long.MIN_VALUE); - for (NodeEnvironment.NodePath nodePath : paths) { - FileStore fileStore = nodePath.fileStore; - BigInteger usableBytes = BigInteger.valueOf(fileStore.getUsableSpace()); - assert usableBytes.compareTo(BigInteger.ZERO) >= 0; + // If no better path is chosen, use the one with the most space by default + NodeEnvironment.NodePath bestPath = getPathWithMostFreeSpace(env); - // Deduct estimated reserved bytes from usable space: - Integer count = dataPathToShardCount.get(nodePath.path); - if (count != null) { - usableBytes = usableBytes.subtract(estShardSizeInBytes.multiply(BigInteger.valueOf(count))); - } - if (bestPath == null || usableBytes.compareTo(maxUsableBytes) > 0) { - maxUsableBytes = usableBytes; - bestPath = nodePath; + if (paths.length != 1) { + int shardCount = indexSettings.getNumberOfShards(); + // Maximum number of shards that a path should have for a particular index assuming + // all the shards were assigned to this node. For example, with a node with 4 data + // paths and an index with 9 primary shards, the maximum number of shards per path + // would be 3. + int maxShardsPerPath = Math.floorDiv(shardCount, paths.length) + ((shardCount % paths.length) == 0 ? 0 : 1); + + Map pathToShardCount = env.shardCountPerPath(shardId.getIndex()); + + // Compute how much space there is on each path + final Map pathsToSpace = new HashMap<>(paths.length); + for (NodeEnvironment.NodePath nodePath : paths) { + FileStore fileStore = nodePath.fileStore; + BigInteger usableBytes = BigInteger.valueOf(fileStore.getUsableSpace()); + pathsToSpace.put(nodePath, usableBytes); } + + bestPath = Arrays.stream(paths) + // Filter out paths that have enough space + .filter((path) -> pathsToSpace.get(path).subtract(estShardSizeInBytes).compareTo(BigInteger.ZERO) > 0) + // Sort by the number of shards for this index + .sorted((p1, p2) -> { + int cmp = Long.compare(pathToShardCount.getOrDefault(p1, 0L), pathToShardCount.getOrDefault(p2, 0L)); + if (cmp == 0) { + // if the number of shards is equal, tie-break with the number of total shards + cmp = Integer.compare(dataPathToShardCount.getOrDefault(p1.path, 0), + dataPathToShardCount.getOrDefault(p2.path, 0)); + if (cmp == 0) { + // if the number of shards is equal, tie-break with the usable bytes + cmp = pathsToSpace.get(p2).compareTo(pathsToSpace.get(p1)); + } + } + return cmp; + }) + // Return the first result + .findFirst() + // Or the existing best path if there aren't any that fit the criteria + .orElse(bestPath); } statePath = bestPath.resolve(shardId); @@ -214,6 +244,24 @@ public final class ShardPath { return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, shardId); } + static NodeEnvironment.NodePath getPathWithMostFreeSpace(NodeEnvironment env) throws IOException { + final NodeEnvironment.NodePath[] paths = env.nodePaths(); + NodeEnvironment.NodePath bestPath = null; + long maxUsableBytes = Long.MIN_VALUE; + for (NodeEnvironment.NodePath nodePath : paths) { + FileStore fileStore = nodePath.fileStore; + long usableBytes = fileStore.getUsableSpace(); + assert usableBytes >= 0 : "usable bytes must be >= 0, got: " + usableBytes; + + if (bestPath == null || usableBytes > maxUsableBytes) { + // This path has been determined to be "better" based on the usable bytes + maxUsableBytes = usableBytes; + bestPath = nodePath; + } + } + return bestPath; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java index 1b960bb1599..e1c88fff3be 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java @@ -36,7 +36,7 @@ import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; import java.io.Closeable; diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index b980e8c9f56..126ed6d92e9 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -53,7 +53,7 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, public static final Setting INDICES_CACHE_QUERY_SIZE_SETTING = Setting.memorySizeSetting("indices.queries.cache.size", "10%", Property.NodeScope); public static final Setting INDICES_CACHE_QUERY_COUNT_SETTING = - Setting.intSetting("indices.queries.cache.count", 10000, 1, Property.NodeScope); + Setting.intSetting("indices.queries.cache.count", 1000, 1, Property.NodeScope); // enables caching on all segments instead of only the larger ones, for testing only public static final Setting INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = Setting.boolSetting("indices.queries.cache.all_segments", false, Property.NodeScope); diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java index c1b46e49567..cec622f4a25 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java @@ -21,6 +21,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.common.Strings; @@ -81,17 +82,21 @@ public class PipelineExecutionService implements ClusterStateApplier { @Override protected void doRun() throws Exception { for (DocWriteRequest actionRequest : actionRequests) { - if ((actionRequest instanceof IndexRequest)) { - IndexRequest indexRequest = (IndexRequest) actionRequest; - if (Strings.hasText(indexRequest.getPipeline())) { - try { - innerExecute(indexRequest, getPipeline(indexRequest.getPipeline())); - //this shouldn't be needed here but we do it for consistency with index api - // which requires it to prevent double execution - indexRequest.setPipeline(null); - } catch (Exception e) { - itemFailureHandler.accept(indexRequest, e); - } + IndexRequest indexRequest = null; + if (actionRequest instanceof IndexRequest) { + indexRequest = (IndexRequest) actionRequest; + } else if (actionRequest instanceof UpdateRequest) { + UpdateRequest updateRequest = (UpdateRequest) actionRequest; + indexRequest = updateRequest.docAsUpsert() ? updateRequest.doc() : updateRequest.upsertRequest(); + } + if (indexRequest != null && Strings.hasText(indexRequest.getPipeline())) { + try { + innerExecute(indexRequest, getPipeline(indexRequest.getPipeline())); + //this shouldn't be needed here but we do it for consistency with index api + // which requires it to prevent double execution + indexRequest.setPipeline(null); + } catch (Exception e) { + itemFailureHandler.accept(indexRequest, e); } } } diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java index f260d7430e2..2c7235ca899 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; import java.util.HashMap; diff --git a/core/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java b/core/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java new file mode 100644 index 00000000000..3deb161cc8e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.node; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Class representing statistics about adaptive replica selection. This includes + * EWMA of queue size, service time, and response time, as well as outgoing + * searches to each node and the "rank" based on the ARS formula. + */ +public class AdaptiveSelectionStats implements Writeable, ToXContentFragment { + + private final Map clientOutgoingConnections; + private final Map nodeComputedStats; + + public AdaptiveSelectionStats(Map clientConnections, + Map nodeComputedStats) { + this.clientOutgoingConnections = clientConnections; + this.nodeComputedStats = nodeComputedStats; + } + + public AdaptiveSelectionStats(StreamInput in) throws IOException { + this.clientOutgoingConnections = in.readMap(StreamInput::readString, StreamInput::readLong); + this.nodeComputedStats = in.readMap(StreamInput::readString, ResponseCollectorService.ComputedNodeStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(this.clientOutgoingConnections, StreamOutput::writeString, StreamOutput::writeLong); + out.writeMap(this.nodeComputedStats, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("adaptive_selection"); + Set allNodeIds = Sets.union(clientOutgoingConnections.keySet(), nodeComputedStats.keySet()); + for (String nodeId : allNodeIds) { + builder.startObject(nodeId); + ResponseCollectorService.ComputedNodeStats stats = nodeComputedStats.get(nodeId); + if (stats != null) { + long outgoingSearches = clientOutgoingConnections.getOrDefault(nodeId, 0L); + builder.field("outgoing_searches", outgoingSearches); + builder.field("avg_queue_size", stats.queueSize); + builder.timeValueField("avg_service_time_ns", "avg_service_time", (long) stats.serviceTime, TimeUnit.NANOSECONDS); + builder.timeValueField("avg_response_time_ns", "avg_response_time", (long) stats.responseTime, TimeUnit.NANOSECONDS); + builder.field("rank", String.format(Locale.ROOT, "%.1f", stats.rank(outgoingSearches))); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + /** + * Returns a map of node id to the outgoing search requests to that node + */ + public Map getOutgoingConnections() { + return clientOutgoingConnections; + } + + /** + * Returns a map of node id to the computed stats + */ + public Map getComputedStats() { + return nodeComputedStats; + } + + /** + * Returns a map of node id to the ranking of the nodes based on the adaptive replica formula + */ + public Map getRanks() { + return nodeComputedStats.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> e.getValue().rank(clientOutgoingConnections.getOrDefault(e.getKey(), 0L)))); + } +} diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 3664ddc62fe..d3d610e4c08 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -451,7 +451,8 @@ public class Node implements Closeable { clusterModule.getAllocationService()); this.nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(), transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(), - httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter()); + httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter(), responseCollectorService, + searchTransportService); modules.add(b -> { b.bind(Node.class).toInstance(this); b.bind(NodeService.class).toInstance(nodeService); diff --git a/core/src/main/java/org/elasticsearch/node/NodeService.java b/core/src/main/java/org/elasticsearch/node/NodeService.java index 319df478f95..5fd4599df94 100644 --- a/core/src/main/java/org/elasticsearch/node/NodeService.java +++ b/core/src/main/java/org/elasticsearch/node/NodeService.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; @@ -36,6 +37,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.monitor.MonitorService; +import org.elasticsearch.node.ResponseCollectorService; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; @@ -54,17 +56,19 @@ public class NodeService extends AbstractComponent implements Closeable { private final CircuitBreakerService circuitBreakerService; private final IngestService ingestService; private final SettingsFilter settingsFilter; - private ScriptService scriptService; + private final ScriptService scriptService; private final HttpServerTransport httpServerTransport; - + private final ResponseCollectorService responseCollectorService; + private final SearchTransportService searchTransportService; private final Discovery discovery; NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery, - TransportService transportService, IndicesService indicesService, PluginsService pluginService, - CircuitBreakerService circuitBreakerService, ScriptService scriptService, - @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService, - SettingsFilter settingsFilter) { + TransportService transportService, IndicesService indicesService, PluginsService pluginService, + CircuitBreakerService circuitBreakerService, ScriptService scriptService, + @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService, + SettingsFilter settingsFilter, ResponseCollectorService responseCollectorService, + SearchTransportService searchTransportService) { super(settings); this.threadPool = threadPool; this.monitorService = monitorService; @@ -77,6 +81,8 @@ public class NodeService extends AbstractComponent implements Closeable { this.ingestService = ingestService; this.settingsFilter = settingsFilter; this.scriptService = scriptService; + this.responseCollectorService = responseCollectorService; + this.searchTransportService = searchTransportService; clusterService.addStateApplier(ingestService.getPipelineStore()); clusterService.addStateApplier(ingestService.getPipelineExecutionService()); } @@ -99,7 +105,7 @@ public class NodeService extends AbstractComponent implements Closeable { public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool, boolean fs, boolean transport, boolean http, boolean circuitBreaker, - boolean script, boolean discoveryStats, boolean ingest) { + boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) return new NodeStats(transportService.getLocalNode(), System.currentTimeMillis(), @@ -114,7 +120,8 @@ public class NodeService extends AbstractComponent implements Closeable { circuitBreaker ? circuitBreakerService.stats() : null, script ? scriptService.stats() : null, discoveryStats ? discovery.stats() : null, - ingest ? ingestService.getPipelineExecutionService().stats() : null + ingest ? ingestService.getPipelineExecutionService().stats() : null, + adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getClientConnections()) : null ); } diff --git a/core/src/main/java/org/elasticsearch/node/ResponseCollectorService.java b/core/src/main/java/org/elasticsearch/node/ResponseCollectorService.java index 6fea0e2e1c0..9881d4404af 100644 --- a/core/src/main/java/org/elasticsearch/node/ResponseCollectorService.java +++ b/core/src/main/java/org/elasticsearch/node/ResponseCollectorService.java @@ -26,9 +26,13 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ExponentiallyWeightedMovingAverage; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -91,6 +95,10 @@ public final class ResponseCollectorService extends AbstractComponent implements return nodeStats; } + public AdaptiveSelectionStats getAdaptiveStats(Map clientSearchConnections) { + return new AdaptiveSelectionStats(clientSearchConnections, getAllNodeStatistics()); + } + /** * Optionally return a {@code NodeStatistics} for the given nodeid, if * response information exists for the given node. Returns an empty @@ -106,7 +114,7 @@ public final class ResponseCollectorService extends AbstractComponent implements * node's statistics. This includes the EWMA of queue size, response time, * and service time. */ - public static class ComputedNodeStats { + public static class ComputedNodeStats implements Writeable { // We store timestamps with nanosecond precision, however, the // formula specifies milliseconds, therefore we need to convert // the values so the times don't unduely weight the formula @@ -120,12 +128,34 @@ public final class ResponseCollectorService extends AbstractComponent implements public final double responseTime; public final double serviceTime; - ComputedNodeStats(int clientNum, NodeStatistics nodeStats) { + public ComputedNodeStats(String nodeId, int clientNum, int queueSize, double responseTime, double serviceTime) { + this.nodeId = nodeId; this.clientNum = clientNum; - this.nodeId = nodeStats.nodeId; - this.queueSize = (int) nodeStats.queueSize.getAverage(); - this.responseTime = nodeStats.responseTime.getAverage(); - this.serviceTime = nodeStats.serviceTime; + this.queueSize = queueSize; + this.responseTime = responseTime; + this.serviceTime = serviceTime; + } + + ComputedNodeStats(int clientNum, NodeStatistics nodeStats) { + this(nodeStats.nodeId, clientNum, + (int) nodeStats.queueSize.getAverage(), nodeStats.responseTime.getAverage(), nodeStats.serviceTime); + } + + ComputedNodeStats(StreamInput in) throws IOException { + this.nodeId = in.readString(); + this.clientNum = in.readInt(); + this.queueSize = in.readInt(); + this.responseTime = in.readDouble(); + this.serviceTime = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.nodeId); + out.writeInt(this.clientNum); + out.writeInt(this.queueSize); + out.writeDouble(this.responseTime); + out.writeDouble(this.serviceTime); } /** @@ -133,9 +163,9 @@ public final class ResponseCollectorService extends AbstractComponent implements * https://www.usenix.org/system/files/conference/nsdi15/nsdi15-paper-suresh.pdf */ private double innerRank(long outstandingRequests) { - // this is a placeholder value, the concurrency compensation is - // defined as the number of outstanding requests from the client - // to the node times the number of clients in the system + // the concurrency compensation is defined as the number of + // outstanding requests from the client to the node times the number + // of clients in the system double concurrencyCompensation = outstandingRequests * clientNum; // Cubic queue adjustment factor. The paper chose 3 though we could diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index a41f3bb4d3e..1a50bcc7213 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -326,6 +327,9 @@ public class PluginsService extends AbstractComponent { try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { for (Path plugin : stream) { + if (FileSystemUtils.isDesktopServicesStore(plugin)) { + continue; + } logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); final PluginInfo info; try { diff --git a/core/src/main/java/org/elasticsearch/rest/action/AcknowledgedRestListener.java b/core/src/main/java/org/elasticsearch/rest/action/AcknowledgedRestListener.java index e12329f93a3..9f08c43fa0f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/AcknowledgedRestListener.java +++ b/core/src/main/java/org/elasticsearch/rest/action/AcknowledgedRestListener.java @@ -36,6 +36,7 @@ public class AcknowledgedRestListener extends Re @Override public RestResponse buildResponse(T response, XContentBuilder builder) throws Exception { + // TODO - Once AcknowledgedResponse implements ToXContent, this method should be updated to call response.toXContent. builder.startObject() .field(Fields.ACKNOWLEDGED, response.isAcknowledged()); addCustomFields(builder, response); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index f6dc8a0a39b..4ff3a65553a 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -93,7 +93,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.TransportRequest; diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 2f67bb37e62..4596162970b 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -301,8 +301,6 @@ public class FetchPhase implements SearchPhase { } context.lookup().source().setSource(nestedSourceAsMap); XContentType contentType = tuple.v1(); - BytesReference nestedSource = contentBuilder(contentType).map(nestedSourceAsMap).bytes(); - context.lookup().source().setSource(nestedSource); context.lookup().source().setSourceContentType(contentType); } return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java index 3171ca4b008..da593d57b77 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java @@ -20,13 +20,20 @@ package org.elasticsearch.search.fetch.subphase; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; public final class FetchSourceSubPhase implements FetchSubPhase { @@ -35,22 +42,27 @@ public final class FetchSourceSubPhase implements FetchSubPhase { if (context.sourceRequested() == false) { return; } + final boolean nestedHit = hitContext.hit().getNestedIdentity() != null; SourceLookup source = context.lookup().source(); FetchSourceContext fetchSourceContext = context.fetchSourceContext(); assert fetchSourceContext.fetchSource(); - if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) { - hitContext.hit().sourceRef(source.internalSourceRef()); - return; + if (nestedHit == false) { + if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) { + hitContext.hit().sourceRef(source.internalSourceRef()); + return; + } + if (source.internalSourceRef() == null) { + throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " + + "for index [" + context.indexShard().shardId().getIndexName() + "]"); + } } - if (source.internalSourceRef() == null) { - throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " + - "for index [" + context.indexShard().shardId().getIndexName() + "]"); + Object value = source.filter(fetchSourceContext); + if (nestedHit) { + value = getNestedSource((Map) value, hitContext); } - - final Object value = source.filter(fetchSourceContext); try { - final int initialCapacity = Math.min(1024, source.internalSourceRef().length()); + final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length()); BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput); builder.value(value); @@ -58,6 +70,12 @@ public final class FetchSourceSubPhase implements FetchSubPhase { } catch (IOException e) { throw new ElasticsearchException("Error filtering source", e); } + } + private Map getNestedSource(Map sourceAsMap, HitContext hitContext) { + for (SearchHit.NestedIdentity o = hitContext.hit().getNestedIdentity(); o != null; o = o.getChild()) { + sourceAsMap = (Map) sourceAsMap.get(o.getField().string()); + } + return sourceAsMap; } } diff --git a/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java index 46b03c61ca0..0c14ef6a815 100644 --- a/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java +++ b/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java @@ -283,9 +283,10 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { return new ScrollingTopDocsCollectorContext(searchContext.scrollContext(), searchContext.sort(), numDocs, searchContext.trackScores(), searchContext.numberOfShards()); } else if (searchContext.collapse() != null) { + boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores(); int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs); return new CollapsingTopDocsCollectorContext(searchContext.collapse(), - searchContext.sort(), numDocs, searchContext.trackScores()); + searchContext.sort(), numDocs, trackScores); } else { int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs); final boolean rescore = searchContext.rescore().isEmpty() == false; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java index 32d4feb4b27..a1c41e40151 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java @@ -57,7 +57,10 @@ public abstract class WordScorer { final long vocSize = terms.getSumTotalTermFreq(); this.vocabluarySize = vocSize == -1 ? reader.maxDoc() : vocSize; this.useTotalTermFreq = vocSize != -1; - this.numTerms = terms.size(); + long numTerms = terms.size(); + // -1 cannot be used as value, because scoreUnigram(...) can then divide by 0 if vocabluarySize is 1. + // -1 is returned when terms is a MultiTerms instance. + this.numTerms = vocabluarySize + numTerms > 1 ? numTerms : 0; this.termsEnum = new FreqTermsEnum(reader, field, !useTotalTermFreq, useTotalTermFreq, null, BigArrays.NON_RECYCLING_INSTANCE); // non recycling for now this.reader = reader; this.realWordLikelyhood = realWordLikelyHood; diff --git a/core/src/main/java/org/elasticsearch/threadpool/Scheduler.java b/core/src/main/java/org/elasticsearch/threadpool/Scheduler.java new file mode 100644 index 00000000000..2901fc1f7a8 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/threadpool/Scheduler.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.threadpool; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsAbortPolicy; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Scheduler that allows to schedule one-shot and periodic commands. + */ +public interface Scheduler { + + static ScheduledThreadPoolExecutor initScheduler(Settings settings) { + ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, + EsExecutors.daemonThreadFactory(settings, "scheduler"), new EsAbortPolicy()); + scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + scheduler.setRemoveOnCancelPolicy(true); + return scheduler; + } + + static boolean terminate(ScheduledThreadPoolExecutor scheduledThreadPoolExecutor, long timeout, TimeUnit timeUnit) { + scheduledThreadPoolExecutor.shutdown(); + if (awaitTermination(scheduledThreadPoolExecutor, timeout, timeUnit)) { + return true; + } + // last resort + scheduledThreadPoolExecutor.shutdownNow(); + return awaitTermination(scheduledThreadPoolExecutor, timeout, timeUnit); + } + + static boolean awaitTermination(final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor, + final long timeout, final TimeUnit timeUnit) { + try { + if (scheduledThreadPoolExecutor.awaitTermination(timeout, timeUnit)) { + return true; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return false; + } + + /** + * Does nothing by default but can be used by subclasses to save the current thread context and wraps the command in a Runnable + * that restores that context before running the command. + */ + default Runnable preserveContext(Runnable command) { + return command; + } + + /** + * Schedules a one-shot command to be run after a given delay. The command is not run in the context of the calling thread. + * To preserve the context of the calling thread you may call {@link #preserveContext(Runnable)} on the runnable before passing + * it to this method. + * The command runs on scheduler thread. Do not run blocking calls on the scheduler thread. Subclasses may allow + * to execute on a different executor, in which case blocking calls are allowed. + * + * @param delay delay before the task executes + * @param executor the name of the executor that has to execute this task. Ignored in the default implementation but can be used + * by subclasses that support multiple executors. + * @param command the command to run + * @return a ScheduledFuture who's get will return when the task has been added to its target thread pool and throws an exception if + * the task is canceled before it was added to its target thread pool. Once the task has been added to its target thread pool + * the ScheduledFuture cannot interact with it. + * @throws EsRejectedExecutionException if the task cannot be scheduled for execution + */ + ScheduledFuture schedule(TimeValue delay, String executor, Runnable command); + + /** + * Schedules a periodic action that runs on scheduler thread. Do not run blocking calls on the scheduler thread. Subclasses may allow + * to execute on a different executor, in which case blocking calls are allowed. + * + * @param command the action to take + * @param interval the delay interval + * @param executor the name of the executor that has to execute this task. Ignored in the default implementation but can be used + * by subclasses that support multiple executors. + * @return a {@link Cancellable} that can be used to cancel the subsequent runs of the command. If the command is running, it will + * not be interrupted. + */ + default Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval, String executor) { + return new ReschedulingRunnable(command, interval, executor, this, (e) -> {}, (e) -> {}); + } + + /** + * This interface represents an object whose execution may be cancelled during runtime. + */ + interface Cancellable { + + /** + * Cancel the execution of this object. This method is idempotent. + */ + void cancel(); + + /** + * Check if the execution has been cancelled + * @return true if cancelled + */ + boolean isCancelled(); + } + + /** + * This class encapsulates the scheduling of a {@link Runnable} that needs to be repeated on a interval. For example, checking a value + * for cleanup every second could be done by passing in a Runnable that can perform the check and the specified interval between + * executions of this runnable. NOTE: the runnable is only rescheduled to run again after completion of the runnable. + * + * For this class, completion means that the call to {@link Runnable#run()} returned or an exception was thrown and caught. In + * case of an exception, this class will log the exception and reschedule the runnable for its next execution. This differs from the + * {@link ScheduledThreadPoolExecutor#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)} semantics as an exception there would + * terminate the rescheduling of the runnable. + */ + final class ReschedulingRunnable extends AbstractRunnable implements Cancellable { + + private final Runnable runnable; + private final TimeValue interval; + private final String executor; + private final Scheduler scheduler; + private final Consumer rejectionConsumer; + private final Consumer failureConsumer; + + private volatile boolean run = true; + + /** + * Creates a new rescheduling runnable and schedules the first execution to occur after the interval specified + * + * @param runnable the {@link Runnable} that should be executed periodically + * @param interval the time interval between executions + * @param executor the executor where this runnable should be scheduled to run + * @param scheduler the {@link Scheduler} instance to use for scheduling + */ + ReschedulingRunnable(Runnable runnable, TimeValue interval, String executor, Scheduler scheduler, + Consumer rejectionConsumer, Consumer failureConsumer) { + this.runnable = runnable; + this.interval = interval; + this.executor = executor; + this.scheduler = scheduler; + this.rejectionConsumer = rejectionConsumer; + this.failureConsumer = failureConsumer; + scheduler.schedule(interval, executor, this); + } + + @Override + public void cancel() { + run = false; + } + + @Override + public boolean isCancelled() { + return run == false; + } + + @Override + public void doRun() { + // always check run here since this may have been cancelled since the last execution and we do not want to run + if (run) { + runnable.run(); + } + } + + @Override + public void onFailure(Exception e) { + failureConsumer.accept(e); + } + + @Override + public void onRejection(Exception e) { + run = false; + rejectionConsumer.accept(e); + } + + @Override + public void onAfter() { + // if this has not been cancelled reschedule it to run again + if (run) { + try { + scheduler.schedule(interval, executor, this); + } catch (final EsRejectedExecutionException e) { + onRejection(e); + } + } + } + } +} diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index a05ffd117ab..e179650aeef 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -33,10 +33,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.EsAbortPolicy; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.XRejectedExecutionHandler; @@ -64,7 +61,7 @@ import java.util.concurrent.TimeUnit; import static java.util.Collections.unmodifiableMap; -public class ThreadPool extends AbstractComponent implements Closeable { +public class ThreadPool extends AbstractComponent implements Scheduler, Closeable { public static class Names { public static final String SAME = "same"; @@ -143,8 +140,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { private Map executors = new HashMap<>(); - private final ScheduledThreadPoolExecutor scheduler; - private final CachedTimeThread cachedTimeThread; static final ExecutorService DIRECT_EXECUTOR = EsExecutors.newDirectExecutorService(); @@ -153,6 +148,8 @@ public class ThreadPool extends AbstractComponent implements Closeable { private final Map builders; + private final ScheduledThreadPoolExecutor scheduler; + public Collection builders() { return Collections.unmodifiableCollection(builders.values()); } @@ -210,12 +207,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { executors.put(Names.SAME, new ExecutorHolder(DIRECT_EXECUTOR, new Info(Names.SAME, ThreadPoolType.DIRECT))); this.executors = unmodifiableMap(executors); - - this.scheduler = new ScheduledThreadPoolExecutor(1, EsExecutors.daemonThreadFactory(settings, "scheduler"), new EsAbortPolicy()); - this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - this.scheduler.setRemoveOnCancelPolicy(true); - + this.scheduler = Scheduler.initScheduler(settings); TimeValue estimatedTimeInterval = ESTIMATED_TIME_INTERVAL_SETTING.get(settings); this.cachedTimeThread = new CachedTimeThread(EsExecutors.threadName(settings, "[timer]"), estimatedTimeInterval.millis()); this.cachedTimeThread.start(); @@ -329,25 +321,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { return holder.executor(); } - public ScheduledExecutorService scheduler() { - return this.scheduler; - } - - /** - * Schedules a periodic action that runs on the specified thread pool. - * - * @param command the action to take - * @param interval the delay interval - * @param executor The name of the thread pool on which to execute this task. {@link Names#SAME} means "execute on the scheduler thread", - * which there is only one of. Executing blocking or long running code on the {@link Names#SAME} thread pool should never - * be done as it can cause issues with the cluster - * @return a {@link Cancellable} that can be used to cancel the subsequent runs of the command. If the command is running, it will - * not be interrupted. - */ - public Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval, String executor) { - return new ReschedulingRunnable(command, interval, executor, this); - } - /** * Schedules a one-shot command to run after a given delay. The command is not run in the context of the calling thread. To preserve the * context of the calling thread you may call threadPool.getThreadContext().preserveContext on the runnable before passing @@ -361,13 +334,30 @@ public class ThreadPool extends AbstractComponent implements Closeable { * @return a ScheduledFuture who's get will return when the task is has been added to its target thread pool and throw an exception if * the task is canceled before it was added to its target thread pool. Once the task has been added to its target thread pool * the ScheduledFuture will cannot interact with it. - * @throws EsRejectedExecutionException if the task cannot be scheduled for execution + * @throws org.elasticsearch.common.util.concurrent.EsRejectedExecutionException if the task cannot be scheduled for execution */ public ScheduledFuture schedule(TimeValue delay, String executor, Runnable command) { if (!Names.SAME.equals(executor)) { command = new ThreadedRunnable(command, executor(executor)); } - return scheduler.schedule(new LoggingRunnable(command), delay.millis(), TimeUnit.MILLISECONDS); + return scheduler.schedule(new ThreadPool.LoggingRunnable(command), delay.millis(), TimeUnit.MILLISECONDS); + } + + @Override + public Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval, String executor) { + return new ReschedulingRunnable(command, interval, executor, this, + (e) -> { + if (logger.isDebugEnabled()) { + logger.debug((Supplier) () -> new ParameterizedMessage("scheduled task [{}] was rejected on thread pool [{}]", + command, executor), e); + } + }, + (e) -> logger.warn((Supplier) () -> new ParameterizedMessage("failed to run scheduled task [{}] on thread pool [{}]", + command, executor), e)); + } + + public Runnable preserveContext(Runnable command) { + return getThreadContext().preserveContext(command); } public void shutdown() { @@ -376,7 +366,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { scheduler.shutdown(); for (ExecutorHolder executor : executors.values()) { if (executor.executor() instanceof ThreadPoolExecutor) { - ((ThreadPoolExecutor) executor.executor()).shutdown(); + executor.executor().shutdown(); } } } @@ -387,7 +377,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { scheduler.shutdownNow(); for (ExecutorHolder executor : executors.values()) { if (executor.executor() instanceof ThreadPoolExecutor) { - ((ThreadPoolExecutor) executor.executor()).shutdownNow(); + executor.executor().shutdownNow(); } } } @@ -396,14 +386,17 @@ public class ThreadPool extends AbstractComponent implements Closeable { boolean result = scheduler.awaitTermination(timeout, unit); for (ExecutorHolder executor : executors.values()) { if (executor.executor() instanceof ThreadPoolExecutor) { - result &= ((ThreadPoolExecutor) executor.executor()).awaitTermination(timeout, unit); + result &= executor.executor().awaitTermination(timeout, unit); } } - cachedTimeThread.join(unit.toMillis(timeout)); return result; } + public ScheduledExecutorService scheduler() { + return this.scheduler; + } + /** * Constrains a value between minimum and maximum values * (inclusive). @@ -726,7 +719,9 @@ public class ThreadPool extends AbstractComponent implements Closeable { if (pool != null) { try { pool.shutdown(); - if (awaitTermination(pool, timeout, timeUnit)) return true; + if (awaitTermination(pool, timeout, timeUnit)) { + return true; + } // last resort pool.shutdownNow(); return awaitTermination(pool, timeout, timeUnit); @@ -738,11 +733,11 @@ public class ThreadPool extends AbstractComponent implements Closeable { } private static boolean awaitTermination( - final ThreadPool pool, + final ThreadPool threadPool, final long timeout, final TimeUnit timeUnit) { try { - if (pool.awaitTermination(timeout, timeUnit)) { + if (threadPool.awaitTermination(timeout, timeUnit)) { return true; } } catch (InterruptedException e) { @@ -760,102 +755,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { return threadContext; } - /** - * This interface represents an object whose execution may be cancelled during runtime. - */ - public interface Cancellable { - - /** - * Cancel the execution of this object. This method is idempotent. - */ - void cancel(); - - /** - * Check if the execution has been cancelled - * @return true if cancelled - */ - boolean isCancelled(); - } - - /** - * This class encapsulates the scheduling of a {@link Runnable} that needs to be repeated on a interval. For example, checking a value - * for cleanup every second could be done by passing in a Runnable that can perform the check and the specified interval between - * executions of this runnable. NOTE: the runnable is only rescheduled to run again after completion of the runnable. - * - * For this class, completion means that the call to {@link Runnable#run()} returned or an exception was thrown and caught. In - * case of an exception, this class will log the exception and reschedule the runnable for its next execution. This differs from the - * {@link ScheduledThreadPoolExecutor#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)} semantics as an exception there would - * terminate the rescheduling of the runnable. - */ - static final class ReschedulingRunnable extends AbstractRunnable implements Cancellable { - - private final Runnable runnable; - private final TimeValue interval; - private final String executor; - private final ThreadPool threadPool; - - private volatile boolean run = true; - - /** - * Creates a new rescheduling runnable and schedules the first execution to occur after the interval specified - * - * @param runnable the {@link Runnable} that should be executed periodically - * @param interval the time interval between executions - * @param executor the executor where this runnable should be scheduled to run - * @param threadPool the {@link ThreadPool} instance to use for scheduling - */ - ReschedulingRunnable(Runnable runnable, TimeValue interval, String executor, ThreadPool threadPool) { - this.runnable = runnable; - this.interval = interval; - this.executor = executor; - this.threadPool = threadPool; - threadPool.schedule(interval, executor, this); - } - - @Override - public void cancel() { - run = false; - } - - @Override - public boolean isCancelled() { - return run == false; - } - - @Override - public void doRun() { - // always check run here since this may have been cancelled since the last execution and we do not want to run - if (run) { - runnable.run(); - } - } - - @Override - public void onFailure(Exception e) { - threadPool.logger.warn((Supplier) () -> new ParameterizedMessage("failed to run scheduled task [{}] on thread pool [{}]", runnable.toString(), executor), e); - } - - @Override - public void onRejection(Exception e) { - run = false; - if (threadPool.logger.isDebugEnabled()) { - threadPool.logger.debug((Supplier) () -> new ParameterizedMessage("scheduled task [{}] was rejected on thread pool [{}]", runnable, executor), e); - } - } - - @Override - public void onAfter() { - // if this has not been cancelled reschedule it to run again - if (run) { - try { - threadPool.schedule(interval, executor, this); - } catch (final EsRejectedExecutionException e) { - onRejection(e); - } - } - } - } - public static boolean assertNotScheduleThread(String reason) { assert Thread.currentThread().getName().contains("scheduler") == false : "Expected current thread [" + Thread.currentThread() + "] to not be the scheduler thread. Reason: [" + reason + "]"; diff --git a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java index f897fdfa749..54b24a86cc3 100644 --- a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java +++ b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; import java.io.IOException; diff --git a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java index aef354a0495..4352f16c05f 100644 --- a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java +++ b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java @@ -54,6 +54,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.hamcrest.core.IsEqual.equalTo; + public class CollapsingTopDocsCollectorTests extends ESTestCase { private static class SegmentSearcher extends IndexSearcher { private final List ctx; @@ -82,12 +84,15 @@ public class CollapsingTopDocsCollectorTests extends ESTestCase { } void assertSearchCollapse(CollapsingDocValuesProducer dvProducers, boolean numeric) throws IOException { - assertSearchCollapse(dvProducers, numeric, true); - assertSearchCollapse(dvProducers, numeric, false); + assertSearchCollapse(dvProducers, numeric, true, true); + assertSearchCollapse(dvProducers, numeric, true, false); + assertSearchCollapse(dvProducers, numeric, false, true); + assertSearchCollapse(dvProducers, numeric, false, false); } private void assertSearchCollapse(CollapsingDocValuesProducer dvProducers, - boolean numeric, boolean multivalued) throws IOException { + boolean numeric, boolean multivalued, + boolean trackMaxScores) throws IOException { final int numDocs = randomIntBetween(1000, 2000); int maxGroup = randomIntBetween(2, 500); final Directory dir = newDirectory(); @@ -118,14 +123,14 @@ public class CollapsingTopDocsCollectorTests extends ESTestCase { final CollapsingTopDocsCollector collapsingCollector; if (numeric) { collapsingCollector = - CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false); + CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, trackMaxScores); } else { collapsingCollector = - CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false); + CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, trackMaxScores); } TopFieldCollector topFieldCollector = - TopFieldCollector.create(sort, totalHits, true, false, false); + TopFieldCollector.create(sort, totalHits, true, trackMaxScores, trackMaxScores); searcher.search(new MatchAllDocsQuery(), collapsingCollector); searcher.search(new MatchAllDocsQuery(), topFieldCollector); @@ -136,6 +141,11 @@ public class CollapsingTopDocsCollectorTests extends ESTestCase { assertEquals(totalHits, collapseTopFieldDocs.totalHits); assertEquals(totalHits, topDocs.scoreDocs.length); assertEquals(totalHits, topDocs.totalHits); + if (trackMaxScores) { + assertThat(collapseTopFieldDocs.getMaxScore(), equalTo(topDocs.getMaxScore())); + } else { + assertThat(collapseTopFieldDocs.getMaxScore(), equalTo(Float.NaN)); + } Set seen = new HashSet<>(); // collapse field is the last sort @@ -186,14 +196,14 @@ public class CollapsingTopDocsCollectorTests extends ESTestCase { } final CollapseTopFieldDocs[] shardHits = new CollapseTopFieldDocs[subSearchers.length]; - final Weight weight = searcher.createNormalizedWeight(new MatchAllDocsQuery(), false); + final Weight weight = searcher.createNormalizedWeight(new MatchAllDocsQuery(), true); for (int shardIDX = 0; shardIDX < subSearchers.length; shardIDX++) { final SegmentSearcher subSearcher = subSearchers[shardIDX]; final CollapsingTopDocsCollector c; if (numeric) { - c = CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false); + c = CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, trackMaxScores); } else { - c = CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false); + c = CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, trackMaxScores); } subSearcher.search(weight, c); shardHits[shardIDX] = c.getTopDocs(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 338ffe06fb8..d9aed454732 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.discovery.zen.PendingClusterStateStats; +import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.http.HttpStats; import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; import org.elasticsearch.indices.breaker.CircuitBreakerStats; @@ -32,6 +33,8 @@ import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.jvm.JvmStats; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.monitor.process.ProcessStats; +import org.elasticsearch.node.AdaptiveSelectionStats; +import org.elasticsearch.node.ResponseCollectorService; import org.elasticsearch.script.ScriptStats; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -46,6 +49,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomLongBetween; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -278,6 +282,22 @@ public class NodeStatsTests extends ESTestCase { assertEquals(stats.getIngestCount(), deserializedStats.getIngestCount()); } } + AdaptiveSelectionStats adaptiveStats = nodeStats.getAdaptiveSelectionStats(); + AdaptiveSelectionStats deserializedAdaptiveStats = deserializedNodeStats.getAdaptiveSelectionStats(); + if (adaptiveStats == null) { + assertNull(deserializedAdaptiveStats); + } else { + assertEquals(adaptiveStats.getOutgoingConnections(), deserializedAdaptiveStats.getOutgoingConnections()); + assertEquals(adaptiveStats.getRanks(), deserializedAdaptiveStats.getRanks()); + adaptiveStats.getComputedStats().forEach((k, v) -> { + ResponseCollectorService.ComputedNodeStats aStats = adaptiveStats.getComputedStats().get(k); + ResponseCollectorService.ComputedNodeStats bStats = deserializedAdaptiveStats.getComputedStats().get(k); + assertEquals(aStats.nodeId, bStats.nodeId); + assertEquals(aStats.queueSize, bStats.queueSize, 0.01); + assertEquals(aStats.serviceTime, bStats.serviceTime, 0.01); + assertEquals(aStats.responseTime, bStats.responseTime, 0.01); + }); + } } } } @@ -392,8 +412,18 @@ public class NodeStatsTests extends ESTestCase { } ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null; - DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(), - randomInt(), randomInt()) : null) : null; + DiscoveryStats discoveryStats = frequently() + ? new DiscoveryStats( + randomBoolean() + ? new PendingClusterStateStats(randomInt(), randomInt(), randomInt()) + : null, + randomBoolean() + ? new PublishClusterStateStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong()) + : null) + : null; IngestStats ingestStats = null; if (frequently()) { IngestStats.Stats totalStats = new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), @@ -407,8 +437,31 @@ public class NodeStatsTests extends ESTestCase { } ingestStats = new IngestStats(totalStats, statsPerPipeline); } + AdaptiveSelectionStats adaptiveSelectionStats = null; + if (frequently()) { + int numNodes = randomIntBetween(0,10); + Map nodeConnections = new HashMap<>(); + Map nodeStats = new HashMap<>(); + for (int i = 0; i < numNodes; i++) { + String nodeId = randomAlphaOfLengthBetween(3, 10); + // add outgoing connection info + if (frequently()) { + nodeConnections.put(nodeId, randomLongBetween(0, 100)); + } + // add node calculations + if (frequently()) { + ResponseCollectorService.ComputedNodeStats stats = new ResponseCollectorService.ComputedNodeStats(nodeId, + randomIntBetween(1,10), randomIntBetween(0, 2000), + randomDoubleBetween(1.0, 10000000.0, true), + randomDoubleBetween(1.0, 10000000.0, true)); + nodeStats.put(nodeId, stats); + } + } + adaptiveSelectionStats = new AdaptiveSelectionStats(nodeConnections, nodeStats); + } //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet - return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo, - transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, ingestStats); + return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, + fsInfo, transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, + ingestStats, adaptiveSelectionStats); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java index ad03d4b001d..19dd64e6324 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdaterTests.java @@ -23,10 +23,15 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class SettingsUpdaterTests extends ESTestCase { @@ -132,4 +137,30 @@ public class SettingsUpdaterTests extends ESTestCase { assertEquals(clusterState.blocks().global().size(), 0); } + + public void testDeprecationLogging() { + Setting deprecatedSetting = + Setting.simpleString("deprecated.setting", Property.Dynamic, Property.NodeScope, Property.Deprecated); + final Settings settings = Settings.builder().put("deprecated.setting", "foo").build(); + final Set> settingsSet = + Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of(deprecatedSetting)).collect(Collectors.toSet()); + final ClusterSettings clusterSettings = new ClusterSettings(settings, settingsSet); + clusterSettings.addSettingsUpdateConsumer(deprecatedSetting, s -> {}); + final SettingsUpdater settingsUpdater = new SettingsUpdater(clusterSettings); + final ClusterState clusterState = + ClusterState.builder(new ClusterName("foo")).metaData(MetaData.builder().persistentSettings(settings).build()).build(); + + final Settings toApplyDebug = Settings.builder().put("logger.org.elasticsearch", "debug").build(); + final ClusterState afterDebug = settingsUpdater.updateSettings(clusterState, toApplyDebug, Settings.EMPTY); + assertSettingDeprecationsAndWarnings(new Setting[] { deprecatedSetting }); + + final Settings toApplyUnset = Settings.builder().putNull("logger.org.elasticsearch").build(); + final ClusterState afterUnset = settingsUpdater.updateSettings(afterDebug, toApplyUnset, Settings.EMPTY); + assertSettingDeprecationsAndWarnings(new Setting[] { deprecatedSetting }); + + // we also check that if no settings are changed, deprecation logging still occurs + settingsUpdater.updateSettings(afterUnset, toApplyUnset, Settings.EMPTY); + assertSettingDeprecationsAndWarnings(new Setting[] { deprecatedSetting }); + } + } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java index 588659335e4..b0fdae9ca62 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java @@ -20,12 +20,19 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; + public class CreateIndexResponseTests extends ESTestCase { public void testSerialization() throws IOException { @@ -62,4 +69,59 @@ public class CreateIndexResponseTests extends ESTestCase { } } } + + public void testToXContent() { + CreateIndexResponse response = new CreateIndexResponse(true, false, "index_name"); + String output = Strings.toString(response); + assertEquals("{\"acknowledged\":true,\"shards_acknowledged\":false,\"index\":\"index_name\"}", output); + } + + public void testToAndFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + /** + * This test adds random fields and objects to the xContent rendered out to + * ensure we can parse it back to be forward compatible with additions to + * the xContent + */ + public void testFromXContentWithRandomFields() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + + final CreateIndexResponse createIndexResponse = createTestItem(); + + boolean humanReadable = randomBoolean(); + final XContentType xContentType = randomFrom(XContentType.values()); + BytesReference originalBytes = toShuffledXContent(createIndexResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + + BytesReference mutated; + if (addRandomFields) { + mutated = insertRandomFields(xContentType, originalBytes, null, random()); + } else { + mutated = originalBytes; + } + CreateIndexResponse parsedCreateIndexResponse; + try (XContentParser parser = createParser(xContentType.xContent(), mutated)) { + parsedCreateIndexResponse = CreateIndexResponse.fromXContent(parser); + assertNull(parser.nextToken()); + } + + assertEquals(createIndexResponse.index(), parsedCreateIndexResponse.index()); + assertEquals(createIndexResponse.isShardsAcked(), parsedCreateIndexResponse.isShardsAcked()); + assertEquals(createIndexResponse.isAcknowledged(), parsedCreateIndexResponse.isAcknowledged()); + } + + /** + * Returns a random {@link CreateIndexResponse}. + */ + private static CreateIndexResponse createTestItem() throws IOException { + boolean acknowledged = randomBoolean(); + boolean shardsAcked = acknowledged && randomBoolean(); + String index = randomAlphaOfLength(5); + + return new CreateIndexResponse(acknowledged, shardsAcked, index); + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponseTests.java new file mode 100755 index 00000000000..4e036319ad9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexResponseTests.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.delete; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; + +public class DeleteIndexResponseTests extends ESTestCase { + + public void testToXContent() { + DeleteIndexResponse response = new DeleteIndexResponse(true); + String output = Strings.toString(response); + assertEquals("{\"acknowledged\":true}", output); + } + + public void testToAndFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + /** + * This test adds random fields and objects to the xContent rendered out to + * ensure we can parse it back to be forward compatible with additions to + * the xContent + */ + public void testFromXContentWithRandomFields() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + + final DeleteIndexResponse deleteIndexResponse = createTestItem(); + + boolean humanReadable = randomBoolean(); + final XContentType xContentType = randomFrom(XContentType.values()); + BytesReference originalBytes = toShuffledXContent(deleteIndexResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + + BytesReference mutated; + if (addRandomFields) { + mutated = insertRandomFields(xContentType, originalBytes, null, random()); + } else { + mutated = originalBytes; + } + DeleteIndexResponse parsedDeleteIndexResponse; + try (XContentParser parser = createParser(xContentType.xContent(), mutated)) { + parsedDeleteIndexResponse = DeleteIndexResponse.fromXContent(parser); + assertNull(parser.nextToken()); + } + + assertEquals(deleteIndexResponse.isAcknowledged(), parsedDeleteIndexResponse.isAcknowledged()); + } + + /** + * Returns a random {@link DeleteIndexResponse}. + */ + private static DeleteIndexResponse createTestItem() throws IOException { + boolean acknowledged = randomBoolean(); + + return new DeleteIndexResponse(acknowledged); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index d33987c92ad..b625b6c10aa 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -82,12 +82,12 @@ public class TransportRolloverActionTests extends ESTestCase { .settings(settings) .build(); final HashSet conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition); - Set results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L), metaData); + Set results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L, between(1, 10000)), metaData); assertThat(results.size(), equalTo(2)); for (Condition.Result result : results) { assertThat(result.matched, equalTo(true)); } - results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0), metaData); + results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, between(1, 10000)), metaData); assertThat(results.size(), equalTo(2)); for (Condition.Result result : results) { if (result.condition instanceof MaxAgeCondition) { @@ -213,10 +213,10 @@ public class TransportRolloverActionTests extends ESTestCase { private IndicesStatsResponse createIndecesStatResponse(long totalDocs, long primaryDocs) { final CommonStats primaryStats = mock(CommonStats.class); - when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0)); + when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0, between(1, 10000))); final CommonStats totalStats = mock(CommonStats.class); - when(totalStats.getDocs()).thenReturn(new DocsStats(totalDocs, 0)); + when(totalStats.getDocs()).thenReturn(new DocsStats(totalDocs, 0, between(1, 10000))); final IndicesStatsResponse response = mock(IndicesStatsResponse.class); when(response.getPrimaries()).thenReturn(primaryStats); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java index b24c8dca79a..83e9cf89d9c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java @@ -73,7 +73,7 @@ public class TransportShrinkActionTests extends ESTestCase { assertTrue( expectThrows(IllegalStateException.class, () -> TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), state, - (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)) + (i) -> new DocsStats(Integer.MAX_VALUE, between(1, 1000), between(1, 100)), new IndexNameExpressionResolver(Settings.EMPTY)) ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -84,7 +84,7 @@ public class TransportShrinkActionTests extends ESTestCase { ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); TransportShrinkAction.prepareCreateIndexRequest(req, clusterState, - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null, new IndexNameExpressionResolver(Settings.EMPTY)); } ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -106,7 +106,7 @@ public class TransportShrinkActionTests extends ESTestCase { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState, - (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)); + (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), new IndexNameExpressionResolver(Settings.EMPTY)); } public void testShrinkIndexSettings() { @@ -128,7 +128,7 @@ public class TransportShrinkActionTests extends ESTestCase { routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards(); - DocsStats stats = new DocsStats(randomIntBetween(0, (IndexWriter.MAX_DOCS) / numSourceShards), randomIntBetween(1, 1000)); + DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); ShrinkRequest target = new ShrinkRequest("target", indexName); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index 2137b33eb08..fca6ca4fd84 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.template.put; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -32,6 +33,11 @@ import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.Is.is; + public class PutIndexTemplateRequestTests extends ESTestCase { // bwc for #21009 @@ -107,4 +113,21 @@ public class PutIndexTemplateRequestTests extends ESTestCase { assertEquals("template", request.patterns().get(0)); } } + + public void testValidateErrorMessage() throws Exception { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + ActionRequestValidationException withoutNameAndPattern = request.validate(); + assertThat(withoutNameAndPattern.getMessage(), containsString("name is missing")); + assertThat(withoutNameAndPattern.getMessage(), containsString("index patterns are missing")); + + request.name("foo"); + ActionRequestValidationException withoutIndexPatterns = request.validate(); + assertThat(withoutIndexPatterns.validationErrors(), hasSize(1)); + assertThat(withoutIndexPatterns.getMessage(), containsString("index patterns are missing")); + + request.patterns(Collections.singletonList("test-*")); + ActionRequestValidationException noError = request.validate(); + assertThat(noError, is(nullValue())); + } + } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java index 9b37a2b9f4c..4ff5b69ad37 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java @@ -67,7 +67,7 @@ public class BulkProcessorTests extends ESTestCase { final BulkProcessor bulkProcessor; assertNull(threadPool.getThreadContext().getHeader(headerKey)); assertNull(threadPool.getThreadContext().getTransient(transientKey)); - try (ThreadContext.StoredContext ctx = threadPool.getThreadContext().stashContext()) { + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { threadPool.getThreadContext().putHeader(headerKey, headerValue); threadPool.getThreadContext().putTransient(transientKey, transientValue); bulkProcessor = new BulkProcessor(consumer, BackoffPolicy.noBackoff(), new BulkProcessor.Listener() { @@ -82,7 +82,7 @@ public class BulkProcessorTests extends ESTestCase { @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { } - }, 1, bulkSize, new ByteSizeValue(5, ByteSizeUnit.MB), flushInterval, threadPool); + }, 1, bulkSize, new ByteSizeValue(5, ByteSizeUnit.MB), flushInterval, threadPool, () -> {}); } assertNull(threadPool.getThreadContext().getHeader(headerKey)); assertNull(threadPool.getThreadContext().getTransient(transientKey)); diff --git a/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 373173a1fc6..9e0b4f7fee9 100644 --- a/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -24,9 +24,12 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.GroupShardsIterator; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.search.SearchPhaseResult; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchTransportRequest; import org.elasticsearch.test.ESTestCase; @@ -38,11 +41,12 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { - public void testFilterShards() throws InterruptedException { final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), @@ -185,6 +189,7 @@ public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode)); lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode)); + final SearchTransportService searchTransportService = new SearchTransportService(Settings.builder().put("search.remote.connect", false).build(), null, null) { @Override @@ -197,11 +202,11 @@ public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { } }; - final AtomicReference> result = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(1); final OriginalIndices originalIndices = new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()); final GroupShardsIterator shardsIter = - SearchAsyncActionTests.getShardsIter("idx", originalIndices, 2048, randomBoolean(), primaryNode, replicaNode); + SearchAsyncActionTests.getShardsIter("idx", originalIndices, 4096, randomBoolean(), primaryNode, replicaNode); + final ExecutorService executor = Executors.newFixedThreadPool(randomIntBetween(1, Runtime.getRuntime().availableProcessors())); final CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase( logger, searchTransportService, @@ -215,16 +220,38 @@ public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { timeProvider, 0, null, - (iter) -> new SearchPhase("test") { + (iter) -> new InitialSearchPhase("test", null, iter, logger, randomIntBetween(1, 32), executor) { @Override - public void run() throws IOException { - result.set(iter); + void onPhaseDone() { latch.countDown(); - }}); + } + + @Override + void onShardFailure(final int shardIndex, final SearchShardTarget shardTarget, final Exception ex) { + + } + + @Override + void onShardSuccess(final SearchPhaseResult result) { + + } + + @Override + protected void executePhaseOnShard( + final SearchShardIterator shardIt, + final ShardRouting shard, + final SearchActionListener listener) { + if (randomBoolean()) { + listener.onResponse(new SearchPhaseResult() {}); + } else { + listener.onFailure(new Exception("failure")); + } + } + }); canMatchPhase.start(); latch.await(); - + executor.shutdown(); } } diff --git a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 81a6359997d..b84dafb4f6d 100644 --- a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; @@ -242,4 +243,43 @@ public class ExpandSearchPhaseTests extends ESTestCase { assertNotNull(reference.get()); assertEquals(1, mockSearchPhaseContext.phasesExecuted.get()); } + + public void testExpandRequestOptions() throws IOException { + MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + mockSearchPhaseContext.searchTransport = new SearchTransportService( + Settings.builder().put("search.remote.connect", false).build(), null, null) { + + @Override + void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { + final QueryBuilder postFilter = QueryBuilders.existsQuery("foo"); + assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); + assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); + assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); + } + }; + mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() + .collapse( + new CollapseBuilder("someField") + .setInnerHits(new InnerHitBuilder().setName("foobarbaz")) + ) + .postFilter(QueryBuilders.existsQuery("foo"))) + .preference("foobar") + .routing("baz"); + + SearchHits hits = new SearchHits(new SearchHit[0], 1, 1.0f); + InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); + AtomicReference reference = new AtomicReference<>(); + ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r -> + new SearchPhase("test") { + @Override + public void run() throws IOException { + reference.set(mockSearchPhaseContext.buildSearchResponse(r, null)); + } + } + ); + phase.run(); + mockSearchPhaseContext.assertNoFailure(); + assertNotNull(reference.get()); + assertEquals(1, mockSearchPhaseContext.phasesExecuted.get()); + } } diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index b9602f26346..8a9c98395d7 100644 --- a/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -50,6 +50,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -285,6 +287,7 @@ public class SearchAsyncActionTests extends ESTestCase { lookup.put(primaryNode.getId(), new MockConnection(primaryNode)); lookup.put(replicaNode.getId(), new MockConnection(replicaNode)); Map aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)); + final ExecutorService executor = Executors.newFixedThreadPool(randomIntBetween(1, Runtime.getRuntime().availableProcessors())); AbstractSearchAsyncAction asyncAction = new AbstractSearchAsyncAction( "test", @@ -295,7 +298,7 @@ public class SearchAsyncActionTests extends ESTestCase { return lookup.get(node); }, aliasFilters, Collections.emptyMap(), - null, + executor, request, responseListener, shardsIter, @@ -349,6 +352,7 @@ public class SearchAsyncActionTests extends ESTestCase { } else { assertTrue(nodeToContextMap.get(replicaNode).toString(), nodeToContextMap.get(replicaNode).isEmpty()); } + executor.shutdown(); } static GroupShardsIterator getShardsIter(String index, OriginalIndices originalIndices, int numShards, diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index 02c4964af3c..999c348b575 100644 --- a/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -175,7 +175,7 @@ public class SearchResponseTests extends ESTestCase { ShardSearchFailure parsedFailure = parsed.getShardFailures()[i]; ShardSearchFailure originalFailure = failures[i]; assertEquals(originalFailure.index(), parsedFailure.index()); - assertEquals(originalFailure.shard().getNodeId(), parsedFailure.shard().getNodeId()); + assertEquals(originalFailure.shard(), parsedFailure.shard()); assertEquals(originalFailure.shardId(), parsedFailure.shardId()); String originalMsg = originalFailure.getCause().getMessage(); assertEquals(parsedFailure.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" + diff --git a/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java index 9a8c0b1feb1..13625a2bc61 100644 --- a/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; @@ -40,12 +41,14 @@ public class ShardSearchFailureTests extends ESTestCase { public static ShardSearchFailure createTestItem() { String randomMessage = randomAlphaOfLengthBetween(3, 20); Exception ex = new ParsingException(0, 0, randomMessage , new IllegalArgumentException("some bad argument")); - String nodeId = randomAlphaOfLengthBetween(5, 10); - String indexName = randomAlphaOfLengthBetween(5, 10); - String indexUuid = randomAlphaOfLengthBetween(5, 10); - int shardId = randomInt(); - return new ShardSearchFailure(ex, - new SearchShardTarget(nodeId, new ShardId(new Index(indexName, indexUuid), shardId), null, null)); + SearchShardTarget searchShardTarget = null; + if (randomBoolean()) { + String nodeId = randomAlphaOfLengthBetween(5, 10); + String indexName = randomAlphaOfLengthBetween(5, 10); + searchShardTarget = new SearchShardTarget(nodeId, + new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), randomInt()), null, null); + } + return new ShardSearchFailure(ex, searchShardTarget); } public void testFromXContent() throws IOException { @@ -80,10 +83,10 @@ public class ShardSearchFailureTests extends ESTestCase { assertNull(parser.nextToken()); } assertEquals(response.index(), parsed.index()); - assertEquals(response.shard().getNodeId(), parsed.shard().getNodeId()); + assertEquals(response.shard(), parsed.shard()); assertEquals(response.shardId(), parsed.shardId()); - /** + /* * we cannot compare the cause, because it will be wrapped in an outer * ElasticSearchException best effort: try to check that the original * message appears somewhere in the rendered xContent diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index 942d7a222ec..09ff06919e9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -152,11 +152,11 @@ public class DiskUsageTests extends ESTestCase { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null), new NodeStats(new DiscoveryNode("node_2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null), new NodeStats(new DiscoveryNode("node_3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1"); @@ -193,11 +193,11 @@ public class DiskUsageTests extends ESTestCase { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null), new NodeStats(new DiscoveryNode("node_2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null), new NodeStats(new DiscoveryNode("node_3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages); DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1"); diff --git a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java index e0a8a1c1e1c..7d4fc0ae0ed 100644 --- a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.io; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -34,6 +35,8 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Arrays; +import static org.hamcrest.Matchers.equalTo; + /** * Unit tests for {@link org.elasticsearch.common.io.FileSystemUtils}. */ @@ -137,4 +140,16 @@ public class FileSystemUtilsTests extends ESTestCase { assertArrayEquals(expectedBytes, actualBytes); } } + + public void testIsDesktopServicesStoreFile() throws IOException { + final Path path = createTempDir(); + final Path desktopServicesStore = path.resolve(".DS_Store"); + Files.createFile(desktopServicesStore); + assertThat(FileSystemUtils.isDesktopServicesStore(desktopServicesStore), equalTo(Constants.MAC_OS_X)); + + Files.delete(desktopServicesStore); + Files.createDirectory(desktopServicesStore); + assertFalse(FileSystemUtils.isDesktopServicesStore(desktopServicesStore)); + } + } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/TimedRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/TimedRunnableTests.java new file mode 100644 index 00000000000..b61f47e67a3 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/TimedRunnableTests.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.util.concurrent; + +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.equalTo; + +public final class TimedRunnableTests extends ESTestCase { + + public void testTimedRunnableDelegatesToAbstractRunnable() { + final boolean isForceExecution = randomBoolean(); + final AtomicBoolean onAfter = new AtomicBoolean(); + final AtomicReference onRejection = new AtomicReference<>(); + final AtomicReference onFailure = new AtomicReference<>(); + final AtomicBoolean doRun = new AtomicBoolean(); + + final AbstractRunnable runnable = new AbstractRunnable() { + @Override + public boolean isForceExecution() { + return isForceExecution; + } + + @Override + public void onAfter() { + onAfter.set(true); + } + + @Override + public void onRejection(final Exception e) { + onRejection.set(e); + } + + @Override + public void onFailure(final Exception e) { + onFailure.set(e); + } + + @Override + protected void doRun() throws Exception { + doRun.set(true); + } + }; + + final TimedRunnable timedRunnable = new TimedRunnable(runnable); + + assertThat(timedRunnable.isForceExecution(), equalTo(isForceExecution)); + + timedRunnable.onAfter(); + assertTrue(onAfter.get()); + + final Exception rejection = new RejectedExecutionException(); + timedRunnable.onRejection(rejection); + assertThat(onRejection.get(), equalTo(rejection)); + + final Exception failure = new Exception(); + timedRunnable.onFailure(failure); + assertThat(onFailure.get(), equalTo(failure)); + + timedRunnable.run(); + assertTrue(doRun.get()); + } + + public void testTimedRunnableDelegatesRunInFailureCase() { + final AtomicBoolean onAfter = new AtomicBoolean(); + final AtomicReference onFailure = new AtomicReference<>(); + final AtomicBoolean doRun = new AtomicBoolean(); + + final Exception exception = new Exception(); + + final AbstractRunnable runnable = new AbstractRunnable() { + @Override + public void onAfter() { + onAfter.set(true); + } + + @Override + public void onFailure(final Exception e) { + onFailure.set(e); + } + + @Override + protected void doRun() throws Exception { + doRun.set(true); + throw exception; + } + }; + + final TimedRunnable timedRunnable = new TimedRunnable(runnable); + timedRunnable.run(); + assertTrue(doRun.get()); + assertThat(onFailure.get(), equalTo(exception)); + assertTrue(onAfter.get()); + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java index bc0bd430a88..7e5bdbd0174 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java @@ -224,7 +224,7 @@ public class ConstructingObjectParserTests extends ESTestCase { parser.apply(createParser(JsonXContent.jsonXContent, "{}"), null); fail("Expected AssertionError"); } catch (AssertionError e) { - assertEquals("[constructor_args_required] must configure at least on constructor argument. If it doesn't have any it should " + assertEquals("[constructor_args_required] must configure at least one constructor argument. If it doesn't have any it should " + "use ObjectParser instead of ConstructingObjectParser. This is a bug in the parser declaration.", e.getMessage()); } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index bc73d5b3bc3..9693a1baadc 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -705,6 +705,73 @@ public class PublishClusterStateActionTests extends ESTestCase { } } + private void assertPublishClusterStateStats(String description, MockNode node, long expectedFull, long expectedIncompatibleDiffs, + long expectedCompatibleDiffs) { + PublishClusterStateStats stats = node.action.stats(); + assertThat(description + ": full cluster states", stats.getFullClusterStateReceivedCount(), equalTo(expectedFull)); + assertThat(description + ": incompatible cluster state diffs", stats.getIncompatibleClusterStateDiffReceivedCount(), + equalTo(expectedIncompatibleDiffs)); + assertThat(description + ": compatible cluster state diffs", stats.getCompatibleClusterStateDiffReceivedCount(), + equalTo(expectedCompatibleDiffs)); + } + + public void testPublishClusterStateStats() throws Exception { + MockNode nodeA = createMockNode("nodeA").setAsMaster(); + MockNode nodeB = createMockNode("nodeB"); + + assertPublishClusterStateStats("nodeA: initial state", nodeA, 0, 0, 0); + assertPublishClusterStateStats("nodeB: initial state", nodeB, 0, 0, 0); + + // Initial cluster state + ClusterState clusterState = nodeA.clusterState; + + // cluster state update - add nodeB + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(clusterState.nodes()).add(nodeB.discoveryNode).build(); + ClusterState previousClusterState = clusterState; + clusterState = ClusterState.builder(clusterState).nodes(discoveryNodes).incrementVersion().build(); + publishStateAndWait(nodeA.action, clusterState, previousClusterState); + + // Sent as a full cluster state update + assertPublishClusterStateStats("nodeA: after full update", nodeA, 0, 0, 0); + assertPublishClusterStateStats("nodeB: after full update", nodeB, 1, 0, 0); + + // Increment cluster state version + previousClusterState = clusterState; + clusterState = ClusterState.builder(clusterState).incrementVersion().build(); + publishStateAndWait(nodeA.action, clusterState, previousClusterState); + + // Sent, successfully, as a cluster state diff + assertPublishClusterStateStats("nodeA: after successful diff update", nodeA, 0, 0, 0); + assertPublishClusterStateStats("nodeB: after successful diff update", nodeB, 1, 0, 1); + + // Increment cluster state version twice + previousClusterState = ClusterState.builder(clusterState).incrementVersion().build(); + clusterState = ClusterState.builder(previousClusterState).incrementVersion().build(); + publishStateAndWait(nodeA.action, clusterState, previousClusterState); + + // Sent, unsuccessfully, as a diff and then retried as a full update + assertPublishClusterStateStats("nodeA: after unsuccessful diff update", nodeA, 0, 0, 0); + assertPublishClusterStateStats("nodeB: after unsuccessful diff update", nodeB, 2, 1, 1); + + // node A steps down from being master + nodeA.resetMasterId(); + nodeB.resetMasterId(); + + // node B becomes the master and sends a version of the cluster state that goes back + discoveryNodes = DiscoveryNodes.builder(discoveryNodes) + .add(nodeA.discoveryNode) + .add(nodeB.discoveryNode) + .masterNodeId(nodeB.discoveryNode.getId()) + .localNodeId(nodeB.discoveryNode.getId()) + .build(); + previousClusterState = ClusterState.builder(new ClusterName("test")).nodes(discoveryNodes).build(); + clusterState = ClusterState.builder(clusterState).nodes(discoveryNodes).incrementVersion().build(); + publishStateAndWait(nodeB.action, clusterState, previousClusterState); + + // Sent, unsuccessfully, as a diff, and then retried as a full update + assertPublishClusterStateStats("nodeA: B became master", nodeA, 1, 1, 0); + assertPublishClusterStateStats("nodeB: B became master", nodeB, 2, 1, 1); + } private MetaData buildMetaDataForVersion(MetaData metaData, long version) { ImmutableOpenMap.Builder indices = ImmutableOpenMap.builder(metaData.indices()); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index 7821d4fd944..ed13f34b609 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -47,7 +47,6 @@ import org.elasticsearch.transport.EmptyTransportResponseHandler; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; -import org.hamcrest.Matchers; import java.io.IOException; import java.net.UnknownHostException; @@ -255,6 +254,11 @@ public class ZenDiscoveryIT extends ESIntegTestCase { " \"total\" : 0,\n" + " \"pending\" : 0,\n" + " \"committed\" : 0\n" + + " },\n" + + " \"published_cluster_states\" : {\n" + + " \"full_states\" : 0,\n" + + " \"incompatible_diffs\" : 0,\n" + + " \"compatible_diffs\" : 0\n" + " }\n" + " }\n" + "}"; @@ -275,6 +279,11 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(stats.getQueueStats().getCommitted(), equalTo(0)); assertThat(stats.getQueueStats().getPending(), equalTo(0)); + assertThat(stats.getPublishStats(), notNullValue()); + assertThat(stats.getPublishStats().getFullClusterStateReceivedCount(), equalTo(0L)); + assertThat(stats.getPublishStats().getIncompatibleClusterStateDiffReceivedCount(), equalTo(0L)); + assertThat(stats.getPublishStats().getCompatibleClusterStateDiffReceivedCount(), equalTo(0L)); + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); builder.startObject(); stats.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java index 26fc15bf662..adc84277a6e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java @@ -35,7 +35,9 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.Arrays; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -155,8 +157,9 @@ public class MultiFieldTests extends ESSingleNodeTestCase { // can to unnecessary re-syncing of the mappings between the local instance and cluster state public void testMultiFieldsInConsistentOrder() throws Exception { String[] multiFieldNames = new String[randomIntBetween(2, 10)]; + Set seenFields = new HashSet<>(); for (int i = 0; i < multiFieldNames.length; i++) { - multiFieldNames[i] = randomAlphaOfLength(4); + multiFieldNames[i] = randomValueOtherThanMany(s -> !seenFields.add(s), () -> randomAlphaOfLength(4)); } XContentBuilder builder = jsonBuilder().startObject().startObject("type").startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 70010cdfc22..5695094553d 100644 --- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; @@ -110,7 +111,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); + new BytesRef("baz"), null, 1f, false, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); assertEquals(expected, actual); } @@ -126,11 +127,11 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); + new BytesRef("baz"), null, 1f, false, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); assertEquals(expected, actual); } - public void testBlendTermsUnsupportedValue() { + public void testBlendTermsUnsupportedValueWithLenient() { FakeFieldType ft1 = new FakeFieldType(); ft1.setName("foo"); FakeFieldType ft2 = new FakeFieldType() { @@ -142,13 +143,29 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz") }; float[] boosts = new float[] {2}; - Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); + Query expected = new DisjunctionMaxQuery(Arrays.asList( + Queries.newMatchNoDocsQuery("failed [" + ft2.name() + "] query, caused by illegal_argument_exception:[null]"), + BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) + ), 1f); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); + new BytesRef("baz"), null, 1f, true, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); assertEquals(expected, actual); } + public void testBlendTermsUnsupportedValueWithoutLenient() { + FakeFieldType ft = new FakeFieldType() { + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new IllegalArgumentException(); + } + }; + ft.setName("bar"); + expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), + new BytesRef("baz"), null, 1f, false, new FieldAndFieldType(ft, 1))); + } + public void testBlendNoTermQuery() { FakeFieldType ft1 = new FakeFieldType(); ft1.setName("foo"); @@ -170,7 +187,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { ), 1.0f); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); + new BytesRef("baz"), null, 1f, false, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); assertEquals(expected, actual); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java b/core/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java new file mode 100644 index 00000000000..85f6764941c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class DocsStatsTests extends ESTestCase { + + public void testCalculateAverageDocSize() throws Exception { + DocsStats stats = new DocsStats(10, 2, 120); + assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); + + stats.add(new DocsStats(0, 0, 0)); + assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); + + stats.add(new DocsStats(8, 30, 480)); + assertThat(stats.getCount(), equalTo(18L)); + assertThat(stats.getDeleted(), equalTo(32L)); + assertThat(stats.getTotalSizeInBytes(), equalTo(600L)); + assertThat(stats.getAverageSizeInBytes(), equalTo(12L)); + } + + public void testSerialize() throws Exception { + DocsStats originalStats = new DocsStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + originalStats.writeTo(out); + BytesReference bytes = out.bytes(); + try (StreamInput in = bytes.streamInput()) { + DocsStats cloneStats = new DocsStats(); + cloneStats.readFrom(in); + assertThat(cloneStats.getCount(), equalTo(originalStats.getCount())); + assertThat(cloneStats.getDeleted(), equalTo(originalStats.getDeleted())); + assertThat(cloneStats.getAverageSizeInBytes(), equalTo(originalStats.getAverageSizeInBytes())); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 5dd7a8f9a40..ddb2108d579 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -67,6 +67,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.VersionType; @@ -88,6 +89,7 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogTests; import org.elasticsearch.indices.IndicesQueryCache; @@ -151,6 +153,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -2229,6 +2232,7 @@ public class IndexShardTests extends IndexShardTestCase { final DocsStats docsStats = indexShard.docStats(); assertThat(docsStats.getCount(), equalTo(numDocs)); assertThat(docsStats.getDeleted(), equalTo(0L)); + assertThat(docsStats.getAverageSizeInBytes(), greaterThan(0L)); } final List ids = randomSubsetOf( @@ -2265,12 +2269,70 @@ public class IndexShardTests extends IndexShardTestCase { final DocsStats docStats = indexShard.docStats(); assertThat(docStats.getCount(), equalTo(numDocs)); assertThat(docStats.getDeleted(), equalTo(0L)); + assertThat(docStats.getAverageSizeInBytes(), greaterThan(0L)); } } finally { closeShards(indexShard); } } + public void testEstimateTotalDocSize() throws Exception { + IndexShard indexShard = null; + try { + indexShard = newStartedShard(true); + + int numDoc = randomIntBetween(100, 200); + for (int i = 0; i < numDoc; i++) { + String doc = XContentFactory.jsonBuilder() + .startObject() + .field("count", randomInt()) + .field("point", randomFloat()) + .field("description", randomUnicodeOfCodepointLength(100)) + .endObject().string(); + indexDoc(indexShard, "doc", Integer.toString(i), doc); + } + + assertThat("Without flushing, segment sizes should be zero", + indexShard.docStats().getTotalSizeInBytes(), equalTo(0L)); + + indexShard.flush(new FlushRequest()); + indexShard.refresh("test"); + { + final DocsStats docsStats = indexShard.docStats(); + final StoreStats storeStats = indexShard.storeStats(); + assertThat(storeStats.sizeInBytes(), greaterThan(numDoc * 100L)); // A doc should be more than 100 bytes. + + assertThat("Estimated total document size is too small compared with the stored size", + docsStats.getTotalSizeInBytes(), greaterThanOrEqualTo(storeStats.sizeInBytes() * 80/100)); + assertThat("Estimated total document size is too large compared with the stored size", + docsStats.getTotalSizeInBytes(), lessThanOrEqualTo(storeStats.sizeInBytes() * 120/100)); + } + + // Do some updates and deletes, then recheck the correlation again. + for (int i = 0; i < numDoc / 2; i++) { + if (randomBoolean()) { + deleteDoc(indexShard, "doc", Integer.toString(i)); + } else { + indexDoc(indexShard, "doc", Integer.toString(i), "{\"foo\": \"bar\"}"); + } + } + + indexShard.flush(new FlushRequest()); + indexShard.refresh("test"); + { + final DocsStats docsStats = indexShard.docStats(); + final StoreStats storeStats = indexShard.storeStats(); + assertThat("Estimated total document size is too small compared with the stored size", + docsStats.getTotalSizeInBytes(), greaterThanOrEqualTo(storeStats.sizeInBytes() * 80/100)); + assertThat("Estimated total document size is too large compared with the stored size", + docsStats.getTotalSizeInBytes(), lessThanOrEqualTo(storeStats.sizeInBytes() * 120/100)); + } + + } finally { + closeShards(indexShard); + } + } + /** * here we are simulating the scenario that happens when we do async shard fetching from GatewaySerivce while we are finishing * a recovery and concurrently clean files. This should always be possible without any exception. Yet there was a bug where IndexShard diff --git a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java index c3d309b486f..7e0328641e3 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.shard; import org.apache.lucene.mockfile.FilterFileSystemProvider; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.settings.Settings; @@ -33,8 +34,10 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; +import java.math.BigInteger; import java.nio.file.FileStore; import java.nio.file.FileSystem; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.FileStoreAttributeView; @@ -45,6 +48,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + /** Separate test class from ShardPathTests because we need static (BeforeClass) setup to install mock filesystems... */ public class NewPathForShardTests extends ESTestCase { @@ -158,6 +164,10 @@ public class NewPathForShardTests extends ESTestCase { } } + static void createFakeShard(ShardPath path) throws IOException { + Files.createDirectories(path.resolveIndex().getParent()); + } + public void testSelectNewPathForShard() throws Exception { Path path = PathUtils.get(createTempDir().toString()); @@ -199,8 +209,10 @@ public class NewPathForShardTests extends ESTestCase { Map dataPathToShardCount = new HashMap<>(); ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); + createFakeShard(result1); dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1); ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); + createFakeShard(result2); // #11122: this was the original failure: on a node with 2 disks that have nearly equal // free space, we would always allocate all N incoming shards to the one path that @@ -210,4 +222,153 @@ public class NewPathForShardTests extends ESTestCase { nodeEnv.close(); } + + public void testSelectNewPathForShardEvenly() throws Exception { + Path path = PathUtils.get(createTempDir().toString()); + + // Use 2 data paths: + String[] paths = new String[] {path.resolve("a").toString(), + path.resolve("b").toString()}; + + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), path) + .putList(Environment.PATH_DATA_SETTING.getKey(), paths).build(); + NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings)); + + // Make sure all our mocking above actually worked: + NodePath[] nodePaths = nodeEnv.nodePaths(); + assertEquals(2, nodePaths.length); + + assertEquals("mocka", nodePaths[0].fileStore.name()); + assertEquals("mockb", nodePaths[1].fileStore.name()); + + // Path a has lots of free space, but b has little, so new shard should go to a: + aFileStore.usableSpace = 100000; + bFileStore.usableSpace = 10000; + + ShardId shardId = new ShardId("index", "uid1", 0); + ShardPath result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, Collections.emptyMap()); + createFakeShard(result); + // First shard should go to a + assertThat(result.getDataPath().toString(), containsString(aPathPart)); + + shardId = new ShardId("index", "uid1", 1); + result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, Collections.emptyMap()); + createFakeShard(result); + // Second shard should go to b + assertThat(result.getDataPath().toString(), containsString(bPathPart)); + + Map dataPathToShardCount = new HashMap<>(); + shardId = new ShardId("index2", "uid2", 0); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index2", + Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3).build()); + ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result1); + dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1); + shardId = new ShardId("index2", "uid2", 1); + ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result2); + dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result2.getDataPath()), 1); + shardId = new ShardId("index2", "uid2", 2); + ShardPath result3 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result3); + // 2 shards go to 'a' and 1 to 'b' + assertThat(result1.getDataPath().toString(), containsString(aPathPart)); + assertThat(result2.getDataPath().toString(), containsString(bPathPart)); + assertThat(result3.getDataPath().toString(), containsString(aPathPart)); + + nodeEnv.close(); + } + + public void testGettingPathWithMostFreeSpace() throws Exception { + Path path = PathUtils.get(createTempDir().toString()); + + // Use 2 data paths: + String[] paths = new String[] {path.resolve("a").toString(), + path.resolve("b").toString()}; + + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), path) + .putList(Environment.PATH_DATA_SETTING.getKey(), paths).build(); + NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings)); + + aFileStore.usableSpace = 100000; + bFileStore.usableSpace = 1000; + + assertThat(ShardPath.getPathWithMostFreeSpace(nodeEnv), equalTo(nodeEnv.nodePaths()[0])); + + aFileStore.usableSpace = 10000; + bFileStore.usableSpace = 20000; + + assertThat(ShardPath.getPathWithMostFreeSpace(nodeEnv), equalTo(nodeEnv.nodePaths()[1])); + + nodeEnv.close(); + } + + public void testTieBreakWithMostShards() throws Exception { + Path path = PathUtils.get(createTempDir().toString()); + + // Use 2 data paths: + String[] paths = new String[] {path.resolve("a").toString(), + path.resolve("b").toString()}; + + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), path) + .putList(Environment.PATH_DATA_SETTING.getKey(), paths).build(); + NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings)); + + // Make sure all our mocking above actually worked: + NodePath[] nodePaths = nodeEnv.nodePaths(); + assertEquals(2, nodePaths.length); + + assertEquals("mocka", nodePaths[0].fileStore.name()); + assertEquals("mockb", nodePaths[1].fileStore.name()); + + // Path a has lots of free space, but b has little, so new shard should go to a: + aFileStore.usableSpace = 100000; + bFileStore.usableSpace = 10000; + + Map dataPathToShardCount = new HashMap<>(); + + ShardId shardId = new ShardId("index", "uid1", 0); + ShardPath result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); + createFakeShard(result); + // First shard should go to a + assertThat(result.getDataPath().toString(), containsString(aPathPart)); + dataPathToShardCount.compute(NodeEnvironment.shardStatePathToDataPath(result.getDataPath()), (k, v) -> v == null ? 1 : v + 1); + + shardId = new ShardId("index", "uid1", 1); + result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); + createFakeShard(result); + // Second shard should go to b + assertThat(result.getDataPath().toString(), containsString(bPathPart)); + dataPathToShardCount.compute(NodeEnvironment.shardStatePathToDataPath(result.getDataPath()), (k, v) -> v == null ? 1 : v + 1); + + shardId = new ShardId("index2", "uid3", 0); + result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); + createFakeShard(result); + // Shard for new index should go to a + assertThat(result.getDataPath().toString(), containsString(aPathPart)); + dataPathToShardCount.compute(NodeEnvironment.shardStatePathToDataPath(result.getDataPath()), (k, v) -> v == null ? 1 : v + 1); + + shardId = new ShardId("index2", "uid2", 0); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index2", + Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3).build()); + ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result1); + dataPathToShardCount.compute(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), (k, v) -> v == null ? 1 : v + 1); + shardId = new ShardId("index2", "uid2", 1); + ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result2); + dataPathToShardCount.compute(NodeEnvironment.shardStatePathToDataPath(result2.getDataPath()), (k, v) -> v == null ? 1 : v + 1); + shardId = new ShardId("index2", "uid2", 2); + ShardPath result3 = ShardPath.selectNewPathForShard(nodeEnv, shardId, idxSettings, 100, dataPathToShardCount); + createFakeShard(result3); + // 2 shards go to 'b' and 1 to 'a' + assertThat(result1.getDataPath().toString(), containsString(bPathPart)); + assertThat(result2.getDataPath().toString(), containsString(aPathPart)); + assertThat(result3.getDataPath().toString(), containsString(bPathPart)); + + nodeEnv.close(); + } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index da90f8023d2..fcc3c93fc37 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -56,7 +56,7 @@ import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; import org.junit.After; import org.junit.Before; diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index d8367b0d6a6..2a490c1dcf9 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import java.io.IOException; import java.util.ArrayList; diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 2b59f0d421c..654927b19f2 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -36,11 +36,16 @@ import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; +import org.elasticsearch.action.support.replication.TransportReplicationActionTests; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Requests; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; @@ -169,6 +174,43 @@ public class IngestClientIT extends ESIntegTestCase { } } + public void testBulkWithUpsert() throws Exception { + createIndex("index"); + + BytesReference source = jsonBuilder().startObject() + .field("description", "my_pipeline") + .startArray("processors") + .startObject() + .startObject("test") + .endObject() + .endObject() + .endArray() + .endObject().bytes(); + PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + + BulkRequest bulkRequest = new BulkRequest(); + IndexRequest indexRequest = new IndexRequest("index", "type", "1").setPipeline("_id"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "val1"); + bulkRequest.add(indexRequest); + UpdateRequest updateRequest = new UpdateRequest("index", "type", "2"); + updateRequest.doc("{}", Requests.INDEX_CONTENT_TYPE); + updateRequest.upsert("{\"field1\":\"upserted_val\"}", XContentType.JSON).upsertRequest().setPipeline("_id"); + bulkRequest.add(updateRequest); + + BulkResponse response = client().bulk(bulkRequest).actionGet(); + + assertThat(response.getItems().length, equalTo(bulkRequest.requests().size())); + Map inserted = client().prepareGet("index", "type", "1") + .get().getSourceAsMap(); + assertThat(inserted.get("field1"), equalTo("val1")); + assertThat(inserted.get("processed"), equalTo(true)); + Map upserted = client().prepareGet("index", "type", "2") + .get().getSourceAsMap(); + assertThat(upserted.get("field1"), equalTo("upserted_val")); + assertThat(upserted.get("processed"), equalTo(true)); + } + public void test() throws Exception { BytesReference source = jsonBuilder().startObject() .field("description", "my_pipeline") diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java index 48817e52d56..f3e86c532d5 100644 --- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java @@ -22,9 +22,9 @@ package org.elasticsearch.monitor.jvm; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; import java.util.AbstractMap; import java.util.HashSet; diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index c3fd0b19f73..2f4644c8589 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.plugins; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; @@ -27,7 +28,9 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.nio.file.FileSystemException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; @@ -36,6 +39,7 @@ import java.util.Locale; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; @LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") public class PluginsServiceTests extends ESTestCase { @@ -124,6 +128,32 @@ public class PluginsServiceTests extends ESTestCase { assertThat(e, hasToString(containsString(expected))); } + public void testDesktopServicesStoreFiles() throws IOException { + final Path home = createTempDir(); + final Settings settings = + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), home) + .build(); + final Path plugins = home.resolve("plugins"); + Files.createDirectories(plugins); + final Path desktopServicesStore = plugins.resolve(".DS_Store"); + Files.createFile(desktopServicesStore); + if (Constants.MAC_OS_X) { + @SuppressWarnings("unchecked") final PluginsService pluginsService = newPluginsService(settings); + assertNotNull(pluginsService); + } else { + final IllegalStateException e = expectThrows(IllegalStateException.class, () -> newPluginsService(settings)); + assertThat(e, hasToString(containsString("Could not load plugin descriptor for existing plugin [.DS_Store]"))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(FileSystemException.class)); + if (Constants.WINDOWS) { + assertThat(e.getCause(), instanceOf(NoSuchFileException.class)); + } else { + assertThat(e.getCause(), hasToString(containsString("Not a directory"))); + } + } + } + public void testStartupWithRemovingMarker() throws IOException { final Path home = createTempDir(); final Settings settings = diff --git a/core/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java new file mode 100644 index 00000000000..5011946914a --- /dev/null +++ b/core/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.document; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.Version; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.hamcrest.CustomMatcher; +import org.mockito.Mockito; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; + +/** + * Tests for {@link RestBulkAction}. + */ +public class RestBulkActionTests extends ESTestCase { + + public void testBulkPipelineUpsert() throws Exception { + final NodeClient mockClient = mock(NodeClient.class); + final Map params = new HashMap<>(); + params.put("pipeline", "timestamps"); + new RestBulkAction(settings(Version.CURRENT).build(), mock(RestController.class)) + .handleRequest( + new FakeRestRequest.Builder( + xContentRegistry()).withPath("my_index/my_type/_bulk").withParams(params) + .withContent( + new BytesArray( + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"field1\":\"val1\"}\n" + + "{\"update\":{\"_id\":\"2\"}}\n" + + "{\"script\":{\"source\":\"ctx._source.counter++;\"},\"upsert\":{\"field1\":\"upserted_val\"}}\n" + ), + XContentType.JSON + ).withMethod(RestRequest.Method.POST).build(), + mock(RestChannel.class), mockClient + ); + Mockito.verify(mockClient) + .bulk(argThat(new CustomMatcher("Pipeline in upsert request") { + @Override + public boolean matches(final Object item) { + BulkRequest request = (BulkRequest) item; + UpdateRequest update = (UpdateRequest) request.requests().get(1); + return "timestamps".equals(update.upsertRequest().getPipeline()); + } + }), any()); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 3822455b83c..4f8493c0b00 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -729,7 +729,7 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(searchHits.getTotalHits(), equalTo(1L)); assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0)); - assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(1)); + assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(1)); bucket = terms.getBucketByKey("b"); assertThat(bucket.getDocCount(), equalTo(2L)); @@ -738,10 +738,10 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(searchHits.getTotalHits(), equalTo(2L)); assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1)); - assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(2)); + assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(2)); assertThat(searchHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(searchHits.getAt(1).getNestedIdentity().getOffset(), equalTo(0)); - assertThat(extractValue("comments.date", searchHits.getAt(1).getSourceAsMap()), equalTo(3)); + assertThat(extractValue("date", searchHits.getAt(1).getSourceAsMap()), equalTo(3)); bucket = terms.getBucketByKey("c"); assertThat(bucket.getDocCount(), equalTo(1L)); @@ -750,7 +750,7 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(searchHits.getTotalHits(), equalTo(1L)); assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1)); - assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(4)); + assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(4)); } public void testTopHitsInSecondLayerNested() throws Exception { @@ -803,49 +803,49 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(topReviewers.getHits().getHits().length, equalTo(7)); assertThat(topReviewers.getHits().getAt(0).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a")); + assertThat(extractValue("name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a")); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(1).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b")); + assertThat(extractValue("name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b")); assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getOffset(), equalTo(1)); assertThat(topReviewers.getHits().getAt(2).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c")); + assertThat(extractValue("name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c")); assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(2)); assertThat(topReviewers.getHits().getAt(3).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c")); + assertThat(extractValue("name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c")); assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(1)); assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(4).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d")); + assertThat(extractValue("name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d")); assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getOffset(), equalTo(1)); assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getOffset(), equalTo(1)); assertThat(topReviewers.getHits().getAt(5).getId(), equalTo("1")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e")); + assertThat(extractValue("name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e")); assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getOffset(), equalTo(1)); assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getOffset(), equalTo(2)); assertThat(topReviewers.getHits().getAt(6).getId(), equalTo("2")); - assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f")); + assertThat(extractValue("name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f")); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0)); assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers")); @@ -901,7 +901,7 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(field.getValue().toString(), equalTo("5")); assertThat(searchHit.getSourceAsMap().size(), equalTo(1)); - assertThat(extractValue("comments.message", searchHit.getSourceAsMap()), equalTo("some comment")); + assertThat(extractValue("message", searchHit.getSourceAsMap()), equalTo("some comment")); } public void testTopHitsInNested() throws Exception { @@ -934,7 +934,7 @@ public class TopHitsIT extends ESIntegTestCase { for (int j = 0; j < 3; j++) { assertThat(searchHits.getAt(j).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(searchHits.getAt(j).getNestedIdentity().getOffset(), equalTo(0)); - assertThat(extractValue("comments.id", searchHits.getAt(j).getSourceAsMap()), equalTo(0)); + assertThat(extractValue("id", searchHits.getAt(j).getSourceAsMap()), equalTo(0)); HighlightField highlightField = searchHits.getAt(j).getHighlightFields().get("comments.message"); assertThat(highlightField.getFragments().length, equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index b98435c8477..b41ba7a85f7 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -596,9 +596,9 @@ public class InnerHitsIT extends ESIntegTestCase { client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject() .field("message", "quick brown fox") .startArray("comments") - .startObject().field("message", "fox eat quick").endObject() - .startObject().field("message", "fox ate rabbit x y z").endObject() - .startObject().field("message", "rabbit got away").endObject() + .startObject().field("message", "fox eat quick").field("x", "y").endObject() + .startObject().field("message", "fox ate rabbit x y z").field("x", "y").endObject() + .startObject().field("message", "rabbit got away").field("x", "y").endObject() .endArray() .endObject()).get(); refresh(); @@ -614,9 +614,11 @@ public class InnerHitsIT extends ESIntegTestCase { assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L)); - assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()), + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"), equalTo("fox eat quick")); - assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()), + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"), equalTo("fox ate rabbit x y z")); response = client().prepareSearch() @@ -627,9 +629,11 @@ public class InnerHitsIT extends ESIntegTestCase { assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L)); - assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()), + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2)); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"), equalTo("fox eat quick")); - assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()), + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2)); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"), equalTo("fox ate rabbit x y z")); } diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index 37ffda5f46a..fd619b69c9e 100644 --- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -472,6 +472,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase { .setQuery(randomizeType(multiMatchQuery("captain america 15", "full_name", "first_name", "last_name", "category", "skill") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) .analyzer("category") + .lenient(true) .operator(Operator.AND))).get(); assertHitCount(searchResponse, 1L); assertFirstHit(searchResponse, hasId("theone")); @@ -480,6 +481,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase { .setQuery(randomizeType(multiMatchQuery("captain america 15", "full_name", "first_name", "last_name", "category", "skill", "int-field") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) .analyzer("category") + .lenient(true) .operator(Operator.AND))).get(); assertHitCount(searchResponse, 1L); assertFirstHit(searchResponse, hasId("theone")); @@ -488,6 +490,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase { .setQuery(randomizeType(multiMatchQuery("captain america 15", "skill", "full_name", "first_name", "last_name", "category", "int-field") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) .analyzer("category") + .lenient(true) .operator(Operator.AND))).get(); assertHitCount(searchResponse, 1L); assertFirstHit(searchResponse, hasId("theone")); @@ -496,6 +499,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase { searchResponse = client().prepareSearch("test") .setQuery(randomizeType(multiMatchQuery("captain america 15", "first_name", "last_name", "skill") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) + .lenient(true) .analyzer("category"))).get(); assertFirstHit(searchResponse, hasId("theone")); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java index d66fd8596bb..40b2b023334 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.reverse.ReverseStringFilter; import org.apache.lucene.analysis.shingle.ShingleFilter; +import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.analysis.synonym.SolrSynonymParser; import org.apache.lucene.analysis.synonym.SynonymFilter; @@ -38,16 +39,14 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.spell.DirectSpellChecker; import org.apache.lucene.search.spell.SuggestMode; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result; import org.elasticsearch.test.ESTestCase; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.io.StringReader; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -439,4 +438,29 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); assertThat(corrections[1].join(new BytesRef(" ")).utf8ToString(), equalTo("xor the god jewel")); } + + public void testFewDocsEgdeCase() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig())) { + Document document = new Document(); + document.add(new TextField("field", "value", Field.Store.NO)); + iw.addDocument(document); + iw.commit(); + document = new Document(); + document.add(new TextField("other_field", "value", Field.Store.NO)); + iw.addDocument(document); + } + + try (DirectoryReader ir = DirectoryReader.open(dir)) { + WordScorer wordScorer = new StupidBackoffScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.95d, new BytesRef(" "), 0.4f); + NoisyChannelSpellChecker suggester = new NoisyChannelSpellChecker(); + DirectSpellChecker spellchecker = new DirectSpellChecker(); + DirectCandidateGenerator generator = new DirectCandidateGenerator(spellchecker, "field", SuggestMode.SUGGEST_MORE_POPULAR, ir, 0.95, 5); + Result result = suggester.getCorrections(new StandardAnalyzer(), new BytesRef("valeu"), generator, 1, 1, ir, "field", wordScorer, 1, 2); + assertThat(result.corrections.length, equalTo(1)); + assertThat(result.corrections[0].join(space).utf8ToString(), equalTo("value")); + } + } + } + } diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 0f6dcec818f..d9d06c26b7d 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -207,39 +207,4 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { assertThat(ex.getMessage(), containsString("is not shared")); } } - - public void testRepositoryVerificationTimeout() throws Exception { - Client client = client(); - - Settings settings = Settings.builder() - .put("location", randomRepoPath()) - .put("random_control_io_exception_rate", 1.0).build(); - logger.info("--> creating repository that cannot write any files - should fail"); - assertThrows(client.admin().cluster().preparePutRepository("test-repo-1") - .setType("mock").setSettings(settings), - RepositoryVerificationException.class); - - logger.info("--> creating repository that cannot write any files, but suppress verification - should be acked"); - assertAcked(client.admin().cluster().preparePutRepository("test-repo-1") - .setType("mock").setSettings(settings).setVerify(false)); - - logger.info("--> verifying repository"); - assertThrows(client.admin().cluster().prepareVerifyRepository("test-repo-1"), RepositoryVerificationException.class); - - Path location = randomRepoPath(); - - logger.info("--> creating repository"); - try { - client.admin().cluster().preparePutRepository("test-repo-1") - .setType("mock") - .setSettings(Settings.builder() - .put("location", location) - .put("localize_location", true) - ).get(); - fail("RepositoryVerificationException wasn't generated"); - } catch (RepositoryVerificationException ex) { - assertThat(ex.getMessage(), containsString("is not shared")); - } - } - } diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java index dd1f4991f95..da0125d6f65 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java @@ -26,9 +26,9 @@ import org.elasticsearch.common.util.concurrent.BaseFuture; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.ThreadPool.Cancellable; +import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool.Names; -import org.elasticsearch.threadpool.ThreadPool.ReschedulingRunnable; +import org.elasticsearch.threadpool.Scheduler.ReschedulingRunnable; import org.junit.After; import org.junit.Before; @@ -80,7 +80,8 @@ public class ScheduleWithFixedDelayTests extends ESTestCase { Thread.currentThread().interrupt(); } }; - ReschedulingRunnable reschedulingRunnable = new ReschedulingRunnable(runnable, delay, Names.GENERIC, threadPool); + ReschedulingRunnable reschedulingRunnable = new ReschedulingRunnable(runnable, delay, Names.GENERIC, threadPool, + (e) -> {}, (e) -> {}); // this call was made during construction of the runnable verify(threadPool, times(1)).schedule(delay, Names.GENERIC, reschedulingRunnable); @@ -260,7 +261,8 @@ public class ScheduleWithFixedDelayTests extends ESTestCase { } }; Runnable runnable = () -> {}; - ReschedulingRunnable reschedulingRunnable = new ReschedulingRunnable(runnable, delay, Names.GENERIC, threadPool); + ReschedulingRunnable reschedulingRunnable = new ReschedulingRunnable(runnable, delay, Names.GENERIC, + threadPool, (e) -> {}, (e) -> {}); assertTrue(reschedulingRunnable.isCancelled()); } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index ec017cfffb5..7029ba048d0 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -43,6 +43,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.HttpURLConnection; +import java.net.URI; import java.net.URL; import java.net.URLConnection; import java.net.URLDecoder; @@ -385,10 +386,40 @@ class InstallPluginCommand extends EnvironmentAwareCommand { } final String expectedChecksum; try (InputStream in = checksumUrl.openStream()) { - BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - expectedChecksum = checksumReader.readLine(); - if (checksumReader.readLine() != null) { - throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); + /* + * The supported format of the SHA-1 files is a single-line file containing the SHA-1. The supported format of the SHA-512 files + * is a single-line file containing the SHA-512 and the filename, separated by two spaces. For SHA-1, we verify that the hash + * matches, and that the file contains a single line. For SHA-512, we verify that the hash and the filename match, and that the + * file contains a single line. + */ + if (digestAlgo.equals("SHA-1")) { + final BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + expectedChecksum = checksumReader.readLine(); + if (checksumReader.readLine() != null) { + throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); + } + } else { + final BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + final String checksumLine = checksumReader.readLine(); + final String[] fields = checksumLine.split(" {2}"); + if (fields.length != 2) { + throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); + } + expectedChecksum = fields[0]; + final String[] segments = URI.create(urlString).getPath().split("/"); + final String expectedFile = segments[segments.length - 1]; + if (fields[1].equals(expectedFile) == false) { + final String message = String.format( + Locale.ROOT, + "checksum file at [%s] is not for this plugin, expected [%s] but was [%s]", + checksumUrl, + expectedFile, + fields[1]); + throw new UserException(ExitCodes.IO_ERROR, message); + } + if (checksumReader.readLine() != null) { + throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); + } } } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 103baccd2d4..f5f8cdb32ef 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.StringReader; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; @@ -77,6 +78,7 @@ import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -799,8 +801,7 @@ public class InstallPluginCommandTests extends ESTestCase { public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); - assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", - bytes -> MessageDigests.toHexString(digest.digest(bytes))); + assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", checksumAndFilename(digest, url)); } public void testOfficalPlugin() throws Exception { @@ -839,8 +840,7 @@ public class InstallPluginCommandTests extends ESTestCase { public void testMavenSha1Backcompat() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); - MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, - ".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes))); + MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", checksum(digest)); assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); } @@ -848,8 +848,7 @@ public class InstallPluginCommandTests extends ESTestCase { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", - bytes -> MessageDigests.toHexString(digest.digest(bytes)))); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", checksum(digest))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); } @@ -862,12 +861,41 @@ public class InstallPluginCommandTests extends ESTestCase { assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); } - public void testInvalidShaFile() throws Exception { + public void testInvalidShaFileMissingFilename() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", - bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar")); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", checksum(digest))); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); + } + + public void testInvalidShaFileMismatchFilename() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl( + "analysis-icu", + "analysis-icu", + url, + null, + ".sha512", + checksumAndString(digest, " repository-s3-" + Version.CURRENT + ".zip"))); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertThat(e, hasToString(matches("checksum file at \\[.*\\] is not for this plugin"))); + } + + public void testInvalidShaFileContainingExtraLine() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl( + "analysis-icu", + "analysis-icu", + url, + null, + ".sha512", + checksumAndString(digest, " analysis-icu-" + Version.CURRENT + ".zip\nfoobar"))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); } @@ -875,8 +903,13 @@ public class InstallPluginCommandTests extends ESTestCase { public void testSha512Mismatch() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", - bytes -> "foobar")); + assertInstallPluginFromUrl( + "analysis-icu", + "analysis-icu", + url, + null, + ".sha512", + bytes -> "foobar analysis-icu-" + Version.CURRENT + ".zip")); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar")); } @@ -884,8 +917,7 @@ public class InstallPluginCommandTests extends ESTestCase { public void testSha1Mismatch() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, - ".sha1", bytes -> "foobar")); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", bytes -> "foobar")); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar")); } @@ -917,4 +949,18 @@ public class InstallPluginCommandTests extends ESTestCase { MockTerminal terminal = installPlugin(pluginZip, env.v1()); assertTrue(Files.exists(KeyStoreWrapper.keystorePath(env.v2().configFile()))); } + + private Function checksum(final MessageDigest digest) { + return checksumAndString(digest, ""); + } + + private Function checksumAndFilename(final MessageDigest digest, final String url) throws MalformedURLException { + final String[] segments = URI.create(url).getPath().split("/"); + return checksumAndString(digest, " " + segments[segments.length - 1]); + } + + private Function checksumAndString(final MessageDigest digest, final String s) { + return bytes -> MessageDigests.toHexString(digest.digest(bytes)) + s; + } + } diff --git a/docs/java-rest/high-level/apis/bulk.asciidoc b/docs/java-rest/high-level/apis/bulk.asciidoc index 9bbc0b31062..3102e96519e 100644 --- a/docs/java-rest/high-level/apis/bulk.asciidoc +++ b/docs/java-rest/high-level/apis/bulk.asciidoc @@ -125,27 +125,24 @@ The `BulkProcessor` simplifies the usage of the Bulk API by providing a utility class that allows index/update/delete operations to be transparently executed as they are added to the processor. -In order to execute the requests, the `BulkProcessor` requires 3 components: +In order to execute the requests, the `BulkProcessor` requires the following +components: `RestHighLevelClient`:: This client is used to execute the `BulkRequest` and to retrieve the `BulkResponse` `BulkProcessor.Listener`:: This listener is called before and after every `BulkRequest` execution or when a `BulkRequest` failed -`ThreadPool`:: The `BulkRequest` executions are done using threads from this -pool, allowing the `BulkProcessor` to work in a non-blocking manner and to -accept new index/update/delete requests while bulk requests are executing. -Then the `BulkProcessor.Builder` class can be used to build a new `BulkProcessor`: +Then the `BulkProcessor.builder` method can be used to build a new `BulkProcessor`: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-init] -------------------------------------------------- -<1> Create the `ThreadPool` using the given `Settings` -<2> Create the `BulkProcessor.Listener` -<3> This method is called before each execution of a `BulkRequest` -<4> This method is called after each execution of a `BulkRequest` -<5> This method is called when a `BulkRequest` failed -<6> Create the `BulkProcessor` by calling the `build()` method from +<1> Create the `BulkProcessor.Listener` +<2> This method is called before each execution of a `BulkRequest` +<3> This method is called after each execution of a `BulkRequest` +<4> This method is called when a `BulkRequest` failed +<5> Create the `BulkProcessor` by calling the `build()` method from the `BulkProcessor.Builder`. The `RestHighLevelClient.bulkAsync()` method will be used to execute the `BulkRequest` under the hood. @@ -190,7 +187,7 @@ to know if the `BulkResponse` contains errors the failure Once all requests have been added to the `BulkProcessor`, its instance needs to -be closed closed using one of the two available closing methods. +be closed using one of the two available closing methods. The `awaitClose()` method can be used to wait until all requests have been processed or the specified waiting time elapses: @@ -209,3 +206,4 @@ include-tagged::{doc-tests}/CRUDDocumentationIT.java[bulk-processor-close] Both methods flush the requests added to the processor before closing the processor and also forbid any new request to be added to it. + diff --git a/docs/java-rest/high-level/apis/deleteindex.asciidoc b/docs/java-rest/high-level/apis/deleteindex.asciidoc new file mode 100644 index 00000000000..3c0627de49a --- /dev/null +++ b/docs/java-rest/high-level/apis/deleteindex.asciidoc @@ -0,0 +1,76 @@ +[[java-rest-high-delete-index]] +=== Delete Index API + +[[java-rest-high-delete-index-request]] +==== Delete Index Request + +A `DeleteIndexRequest` requires an `index` argument: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-request] +-------------------------------------------------- +<1> Index + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-request-timeout] +-------------------------------------------------- +<1> Timeout to wait for the all the nodes to acknowledge the index deletion as a `TimeValue` +<2> Timeout to wait for the all the nodes to acknowledge the index deletion as a `String` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-request-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-request-indicesOptions] +-------------------------------------------------- +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and +how wildcard expressions are expanded + +[[java-rest-high-delete-index-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-execute] +-------------------------------------------------- + +[[java-rest-high-delete-index-async]] +==== Asynchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-execute-async] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-delete-index-response]] +==== Delete Index Response + +The returned `DeleteIndexResponse` allows to retrieve information about the executed + operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-response] +-------------------------------------------------- +<1> Indicates whether all of the nodes have acknowledged the request or not + +If the index was not found, an `ElasticsearchException` will be thrown: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-notfound] +-------------------------------------------------- +<1> Do something if the index to be deleted was not found diff --git a/docs/java-rest/high-level/apis/index.asciidoc b/docs/java-rest/high-level/apis/index.asciidoc index b4dcf7e9d80..993951b5ae7 100644 --- a/docs/java-rest/high-level/apis/index.asciidoc +++ b/docs/java-rest/high-level/apis/index.asciidoc @@ -1,3 +1,4 @@ +include::deleteindex.asciidoc[] include::_index.asciidoc[] include::get.asciidoc[] include::delete.asciidoc[] diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 51411ea9fca..9e902e17157 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -3,6 +3,9 @@ The Java High Level REST Client supports the following APIs: +Indices APIs:: +* <> + Single document APIs:: * <> * <> diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc index 51274b109ad..a9a663dd99d 100644 --- a/docs/plugins/ingest-geoip.asciidoc +++ b/docs/plugins/ingest-geoip.asciidoc @@ -10,7 +10,7 @@ under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.c The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory, and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed -with gzip. The geoip config directory is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too. +with gzip. The geoip config directory is located at `$ES_HOME/config/ingest-geoip` and holds the shipped databases too. :plugin_name: ingest-geoip include::install_remove.asciidoc[] @@ -187,4 +187,4 @@ The geoip processor supports the following setting: The maximum number of results that should be cached. Defaults to `1000`. -Note that these settings are node settings and apply to all geoip processors, i.e. there is one cache for all defined geoip processors. \ No newline at end of file +Note that these settings are node settings and apply to all geoip processors, i.e. there is one cache for all defined geoip processors. diff --git a/docs/plugins/ingest.asciidoc b/docs/plugins/ingest.asciidoc index 490d3d1362b..fa316c3862f 100644 --- a/docs/plugins/ingest.asciidoc +++ b/docs/plugins/ingest.asciidoc @@ -25,6 +25,13 @@ under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.c A processor that extracts details from the User-Agent header value. +[float] +=== Community contributed ingest plugins + +The following plugin has been contributed by our community: + +* https://github.com/johtani/elasticsearch-ingest-csv[Ingest CSV Processor Plugin] (by Jun Ohtani) + include::ingest-attachment.asciidoc[] include::ingest-geoip.asciidoc[] diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 099559cb18b..7668a0df792 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -329,10 +329,8 @@ Top hits response snippet with a nested hit, which resides in the first slot of }, "_score": 0.2876821, "_source": { - "comments": { - "comment": "This car could have better brakes", <3> - "username": "baddriver007" - } + "comment": "This car could have better brakes", <3> + "username": "baddriver007" } } ] diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 7875f011abe..0ac58622d51 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -70,7 +70,8 @@ type specified. Check out the <> section for more information on mapping definitions. Automatic index creation can be disabled by setting -`action.auto_create_index` to `false` in the config file of all nodes. +`action.auto_create_index` to `false` in the config file of all nodes, +or via the cluster update settings API. Automatic mapping creation can be disabled by setting `index.mapper.dynamic` to `false` per-index as an index setting. diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index 00876365a27..46989857235 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -149,7 +149,7 @@ And now we are ready to start our node and single cluster: -------------------------------------------------- [float] -=== Instalation with Homebrew +=== Installation with Homebrew On macOS, Elasticsearch can also be installed via https://brew.sh[Homebrew]: @@ -298,7 +298,13 @@ epoch timestamp cluster status node.total node.data shards pri relo i We can see that our cluster named "elasticsearch" is up with a green status. -Whenever we ask for the cluster health, we either get green, yellow, or red. Green means everything is good (cluster is fully functional), yellow means all data is available but some replicas are not yet allocated (cluster is fully functional), and red means some data is not available for whatever reason. Note that even if a cluster is red, it still is partially functional (i.e. it will continue to serve search requests from the available shards) but you will likely need to fix it ASAP since you have missing data. +Whenever we ask for the cluster health, we either get green, yellow, or red. + + * Green - everything is good (cluster is fully functional) + * Yellow - all data is available but some replicas are not yet allocated (cluster is fully functional) + * Red - some data is not available for whatever reason (cluster is partially functional) + +**Note:** When a cluster is red, it will continue to serve search requests from the available shards but you will likely need to fix it ASAP since there are unassigned shards. Also from the above response, we can see a total of 1 node and that we have 0 shards since we have no data in it yet. Note that since we are using the default cluster name (elasticsearch) and since Elasticsearch uses unicast network discovery by default to find other nodes on the same machine, it is possible that you could accidentally start up more than one node on your computer and have them all join a single cluster. In this scenario, you may see more than 1 node in the above response. diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc index 6a24ca626d9..2db12a33368 100644 --- a/docs/reference/mapping/params/ignore-above.asciidoc +++ b/docs/reference/mapping/params/ignore-above.asciidoc @@ -56,5 +56,5 @@ limit of `32766`. NOTE: The value for `ignore_above` is the _character count_, but Lucene counts bytes. If you use UTF-8 text with many non-ASCII characters, you may want to -set the limit to `32766 / 3 = 10922` since UTF-8 characters may occupy at most -3 bytes. +set the limit to `32766 / 4 = 8191` since UTF-8 characters may occupy at most +4 bytes. diff --git a/docs/reference/mapping/params/index.asciidoc b/docs/reference/mapping/params/index.asciidoc index e097293d142..32916e98d35 100644 --- a/docs/reference/mapping/params/index.asciidoc +++ b/docs/reference/mapping/params/index.asciidoc @@ -2,5 +2,5 @@ === `index` The `index` option controls whether field values are indexed. It accepts `true` -or `false`. Fields that are not indexed are not queryable. +or `false` and defaults to `true`. Fields that are not indexed are not queryable. diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 8ef7c3b2bf2..2cbc3a5bc54 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -34,7 +34,7 @@ string:: <> and <> <>:: `completion` to provide auto-complete suggestions <>:: `token_count` to count the number of tokens in a string -{plugins}/mapper-size.html[`mapper-murmur3`]:: `murmur3` to compute hashes of values at index-time and store them in the index +{plugins}/mapper-murmur3.html[`mapper-murmur3`]:: `murmur3` to compute hashes of values at index-time and store them in the index <>:: Accepts queries from the query-dsl diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index ad33205650d..56396ce7584 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -114,6 +114,17 @@ PUT my_index/doc/4?routing=1&refresh <2> `answer` is the name of the join for this document <3> The parent id of this child document +==== Parent-join and performance. + +The join field shouldn't be used like joins in a relation database. In Elasticsearch the key to good performance +is to de-normalize your data into documents. Each join field, `has_child` or `has_parent` query adds a +significant tax to your query performance. + +The only case where the join field makes sense is if your data contains a one-to-many relationship where +one entity significantly outnumbers the other entity. An example of such case is a use case with products +and offers for these products. In the case that offers significantly outnumbers the number of products then +it makes sense to model the product as parent document and the offer as child document. + ==== Parent-join restrictions * Only one `join` field mapping is allowed per index. @@ -338,7 +349,7 @@ GET _nodes/stats/indices/fielddata?human&fields=my_join_field#question // CONSOLE // TEST[continued] -==== Multiple levels of parent join +==== Multiple children per parent It is also possible to define multiple children for a single parent: @@ -363,62 +374,3 @@ PUT my_index // CONSOLE <1> `question` is parent of `answer` and `comment`. - -And multiple levels of parent/child: - -[source,js] --------------------------------------------------- -PUT my_index -{ - "mappings": { - "doc": { - "properties": { - "my_join_field": { - "type": "join", - "relations": { - "question": ["answer", "comment"], <1> - "answer": "vote" <2> - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE - -<1> `question` is parent of `answer` and `comment` -<2> `answer` is parent of `vote` - -The mapping above represents the following tree: - - question - / \ - / \ - comment answer - | - | - vote - -Indexing a grand child document requires a `routing` value equals -to the grand-parent (the greater parent of the lineage): - - -[source,js] --------------------------------------------------- -PUT my_index/doc/3?routing=1&refresh <1> -{ - "text": "This is a vote", - "my_join_field": { - "name": "vote", - "parent": "2" <2> - } -} --------------------------------------------------- -// CONSOLE -// TEST[continued] - -<1> This child document must be on the same shard than its grandparent and parent -<2> The parent id of this document (must points to an `answer` document) - - diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc index 62b2f375ee6..eb755564ed5 100644 --- a/docs/reference/modules/cross-cluster-search.asciidoc +++ b/docs/reference/modules/cross-cluster-search.asciidoc @@ -1,8 +1,6 @@ [[modules-cross-cluster-search]] == Cross Cluster Search -beta[] - The _cross cluster search_ feature allows any node to act as a federated client across multiple clusters. In contrast to the <> feature, a cross cluster search node won't join the remote cluster, instead it connects to a remote cluster in a light fashion in order to execute diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc index bfe7eff4c2f..d13ae326fb7 100644 --- a/docs/reference/query-dsl/has-child-query.asciidoc +++ b/docs/reference/query-dsl/has-child-query.asciidoc @@ -23,6 +23,14 @@ GET /_search -------------------------------------------------- // CONSOLE +Note that the `has_child` is a slow query compared to other queries in the +query dsl due to the fact that it performs a join. The performance degrades +as the number of matching child documents pointing to unique parent documents +increases. If you care about query performance you should not use this query. +However if you do happen to use this query then use it as less as possible. Each +`has_child` query that gets added to a search request can increase query time +significantly. + [float] ==== Scoring capabilities diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc index a1dcf605ddd..4065a9d99fe 100644 --- a/docs/reference/query-dsl/has-parent-query.asciidoc +++ b/docs/reference/query-dsl/has-parent-query.asciidoc @@ -25,6 +25,13 @@ GET /_search -------------------------------------------------- // CONSOLE +Note that the `has_parent` is a slow query compared to other queries in the +query dsl due to the fact that it performs a join. The performance degrades +as the number of matching parent documents increases. If you care about query +performance you should not use this query. However if you do happen to use +this query then use it as less as possible. Each `has_parent` query that gets +added to a search request can increase query time significantly. + [float] ==== Scoring capabilities diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index 952cdedd423..a9da7378809 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -158,10 +158,8 @@ An example of a response snippet that could be generated from the above search r }, "_score": 1.0, "_source": { - "comments" : { - "author": "nik9000", - "number": 2 - } + "author": "nik9000", + "number": 2 } } ] @@ -406,12 +404,8 @@ Which would look like: }, "_score": 0.6931472, "_source": { - "comments": { - "votes": { - "value": 1, - "voter": "kimchy" - } - } + "value": 1, + "voter": "kimchy" } } ] diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java index 69e14c14a7c..6ff132b32fa 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java @@ -193,7 +193,7 @@ public class InternalMatrixStatsTests extends InternalAggregationTestCase randomAlphaOfLength(3)); for (MatrixStats matrix : Arrays.asList(actual)) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index e25891aca4e..311f30513c1 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -63,7 +63,12 @@ public final class DateIndexNameProcessor extends AbstractProcessor { @Override public void execute(IngestDocument ingestDocument) throws Exception { // Date can be specified as a string or long: - String date = Objects.toString(ingestDocument.getFieldValue(field, Object.class)); + Object obj = ingestDocument.getFieldValue(field, Object.class); + String date = null; + if (obj != null) { + // Not use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers + date = obj.toString(); + } DateTime dateTime = null; Exception lastException = null; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 4dec8f53d3f..f1e7dcdcf55 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -30,7 +30,6 @@ import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; import java.util.ArrayList; -import java.util.IllformedLocaleException; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,7 +63,12 @@ public final class DateProcessor extends AbstractProcessor { @Override public void execute(IngestDocument ingestDocument) { - String value = ingestDocument.getFieldValue(field, String.class); + Object obj = ingestDocument.getFieldValue(field, Object.class); + String value = null; + if (obj != null) { + // Not use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers + value = obj.toString(); + } DateTime dateTime = null; Exception lastException = null; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index d92f2e84be1..cc68340ec59 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -134,6 +134,12 @@ public class DateProcessorTests extends ESTestCase { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); dateProcessor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("1970-01-01T00:16:40.500Z")); + + document = new HashMap<>(); + document.put("date_as_string", 1000500L); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + dateProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("1970-01-01T00:16:40.500Z")); } public void testUnix() { diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index bc8a118ba1d..0bd96725c66 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -29,7 +29,7 @@ integTestCluster { } dependencies { - compile 'org.antlr:antlr4-runtime:4.5.1-1' + compile 'org.antlr:antlr4-runtime:4.5.3' compile 'org.ow2.asm:asm-debug-all:5.1' } @@ -71,7 +71,7 @@ configurations { } dependencies { - regenerate 'org.antlr:antlr4:4.5.1-1' + regenerate 'org.antlr:antlr4:4.5.3' } String grammarPath = 'src/main/antlr' diff --git a/modules/lang-painless/licenses/antlr4-runtime-4.5.1-1.jar.sha1 b/modules/lang-painless/licenses/antlr4-runtime-4.5.1-1.jar.sha1 deleted file mode 100644 index 37f80b91724..00000000000 --- a/modules/lang-painless/licenses/antlr4-runtime-4.5.1-1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66144204f9d6d7d3f3f775622c2dd7e9bd511d97 \ No newline at end of file diff --git a/modules/lang-painless/licenses/antlr4-runtime-4.5.3.jar.sha1 b/modules/lang-painless/licenses/antlr4-runtime-4.5.3.jar.sha1 new file mode 100644 index 00000000000..535955b7d68 --- /dev/null +++ b/modules/lang-painless/licenses/antlr4-runtime-4.5.3.jar.sha1 @@ -0,0 +1 @@ +2609e36f18f7e8d593cc1cddfb2ac776dc96b8e0 \ No newline at end of file diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java index 430fe0b8cd5..02b7105593f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java @@ -24,34 +24,19 @@ import org.elasticsearch.painless.Definition.Type; import java.util.Objects; -import static org.elasticsearch.painless.Definition.BOOLEAN_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.BOOLEAN_TYPE; -import static org.elasticsearch.painless.Definition.BYTE_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.BYTE_TYPE; -import static org.elasticsearch.painless.Definition.CHAR_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.CHAR_TYPE; -import static org.elasticsearch.painless.Definition.DEF_TYPE; -import static org.elasticsearch.painless.Definition.DOUBLE_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.DOUBLE_TYPE; -import static org.elasticsearch.painless.Definition.FLOAT_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.FLOAT_TYPE; -import static org.elasticsearch.painless.Definition.INT_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.INT_TYPE; -import static org.elasticsearch.painless.Definition.LONG_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.LONG_TYPE; -import static org.elasticsearch.painless.Definition.NUMBER_TYPE; -import static org.elasticsearch.painless.Definition.OBJECT_TYPE; -import static org.elasticsearch.painless.Definition.SHORT_OBJ_TYPE; -import static org.elasticsearch.painless.Definition.SHORT_TYPE; -import static org.elasticsearch.painless.Definition.STRING_TYPE; - /** * Used during the analysis phase to collect legal type casts and promotions * for type-checking and later to write necessary casts in the bytecode. */ public final class AnalyzerCaster { - public static Cast getLegalCast(Location location, Type actual, Type expected, boolean explicit, boolean internal) { + private Definition definition; + + public AnalyzerCaster(Definition definition) { + this.definition = definition; + } + + public Cast getLegalCast(Location location, Type actual, Type expected, boolean explicit, boolean internal) { Objects.requireNonNull(actual); Objects.requireNonNull(expected); @@ -61,421 +46,421 @@ public final class AnalyzerCaster { if (actual.dynamic) { if (expected.clazz == boolean.class) { - return new Cast(DEF_TYPE, BOOLEAN_OBJ_TYPE, explicit, null, BOOLEAN_TYPE, null, null); + return new Cast(definition.DefType, definition.BooleanType, explicit, null, definition.booleanType, null, null); } else if (expected.clazz == byte.class) { - return new Cast(DEF_TYPE, BYTE_OBJ_TYPE, explicit, null, BYTE_TYPE, null, null); + return new Cast(definition.DefType, definition.ByteType, explicit, null, definition.byteType, null, null); } else if (expected.clazz == short.class) { - return new Cast(DEF_TYPE, SHORT_OBJ_TYPE, explicit, null, SHORT_TYPE, null, null); + return new Cast(definition.DefType, definition.ShortType, explicit, null, definition.shortType, null, null); } else if (expected.clazz == char.class) { - return new Cast(DEF_TYPE, CHAR_OBJ_TYPE, explicit, null, CHAR_TYPE, null, null); + return new Cast(definition.DefType, definition.CharacterType, explicit, null, definition.charType, null, null); } else if (expected.clazz == int.class) { - return new Cast(DEF_TYPE, INT_OBJ_TYPE, explicit, null, INT_TYPE, null, null); + return new Cast(definition.DefType, definition.IntegerType, explicit, null, definition.intType, null, null); } else if (expected.clazz == long.class) { - return new Cast(DEF_TYPE, LONG_OBJ_TYPE, explicit, null, LONG_TYPE, null, null); + return new Cast(definition.DefType, definition.LongType, explicit, null, definition.longType, null, null); } else if (expected.clazz == float.class) { - return new Cast(DEF_TYPE, FLOAT_OBJ_TYPE, explicit, null, FLOAT_TYPE, null, null); + return new Cast(definition.DefType, definition.FloatType, explicit, null, definition.floatType, null, null); } else if (expected.clazz == double.class) { - return new Cast(DEF_TYPE, DOUBLE_OBJ_TYPE, explicit, null, DOUBLE_TYPE, null, null); + return new Cast(definition.DefType, definition.DoubleType, explicit, null, definition.doubleType, null, null); } } else if (actual.clazz == Object.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(OBJECT_TYPE, BYTE_OBJ_TYPE, true, null, BYTE_TYPE, null, null); + return new Cast(definition.ObjectType, definition.ByteType, true, null, definition.byteType, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(OBJECT_TYPE, SHORT_OBJ_TYPE, true, null, SHORT_TYPE, null, null); + return new Cast(definition.ObjectType, definition.ShortType, true, null, definition.shortType, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(OBJECT_TYPE, CHAR_OBJ_TYPE, true, null, CHAR_TYPE, null, null); + return new Cast(definition.ObjectType, definition.CharacterType, true, null, definition.charType, null, null); } else if (expected.clazz == int.class && explicit && internal) { - return new Cast(OBJECT_TYPE, INT_OBJ_TYPE, true, null, INT_TYPE, null, null); + return new Cast(definition.ObjectType, definition.IntegerType, true, null, definition.intType, null, null); } else if (expected.clazz == long.class && explicit && internal) { - return new Cast(OBJECT_TYPE, LONG_OBJ_TYPE, true, null, LONG_TYPE, null, null); + return new Cast(definition.ObjectType, definition.LongType, true, null, definition.longType, null, null); } else if (expected.clazz == float.class && explicit && internal) { - return new Cast(OBJECT_TYPE, FLOAT_OBJ_TYPE, true, null, FLOAT_TYPE, null, null); + return new Cast(definition.ObjectType, definition.FloatType, true, null, definition.floatType, null, null); } else if (expected.clazz == double.class && explicit && internal) { - return new Cast(OBJECT_TYPE, DOUBLE_OBJ_TYPE, true, null, DOUBLE_TYPE, null, null); + return new Cast(definition.ObjectType, definition.DoubleType, true, null, definition.doubleType, null, null); } } else if (actual.clazz == Number.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(NUMBER_TYPE, BYTE_OBJ_TYPE, true, null, BYTE_TYPE, null, null); + return new Cast(definition.NumberType, definition.ByteType, true, null, definition.byteType, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(NUMBER_TYPE, SHORT_OBJ_TYPE, true, null, SHORT_TYPE, null, null); + return new Cast(definition.NumberType, definition.ShortType, true, null, definition.shortType, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(NUMBER_TYPE, CHAR_OBJ_TYPE, true, null, CHAR_TYPE, null, null); + return new Cast(definition.NumberType, definition.CharacterType, true, null, definition.charType, null, null); } else if (expected.clazz == int.class && explicit && internal) { - return new Cast(NUMBER_TYPE, INT_OBJ_TYPE, true, null, INT_TYPE, null, null); + return new Cast(definition.NumberType, definition.IntegerType, true, null, definition.intType, null, null); } else if (expected.clazz == long.class && explicit && internal) { - return new Cast(NUMBER_TYPE, LONG_OBJ_TYPE, true, null, LONG_TYPE, null, null); + return new Cast(definition.NumberType, definition.LongType, true, null, definition.longType, null, null); } else if (expected.clazz == float.class && explicit && internal) { - return new Cast(NUMBER_TYPE, FLOAT_OBJ_TYPE, true, null, FLOAT_TYPE, null, null); + return new Cast(definition.NumberType, definition.FloatType, true, null, definition.floatType, null, null); } else if (expected.clazz == double.class && explicit && internal) { - return new Cast(NUMBER_TYPE, DOUBLE_OBJ_TYPE, true, null, DOUBLE_TYPE, null, null); + return new Cast(definition.NumberType, definition.DoubleType, true, null, definition.doubleType, null, null); } } else if (actual.clazz == String.class) { if (expected.clazz == char.class && explicit) { - return new Cast(STRING_TYPE, CHAR_TYPE, true); + return new Cast(definition.StringType, definition.charType, true); } } else if (actual.clazz == boolean.class) { if (expected.dynamic) { - return new Cast(BOOLEAN_OBJ_TYPE, DEF_TYPE, explicit, null, null, BOOLEAN_TYPE, null); + return new Cast(definition.BooleanType, definition.DefType, explicit, null, null, definition.booleanType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(BOOLEAN_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, BOOLEAN_TYPE, null); + return new Cast(definition.BooleanType, definition.ObjectType, explicit, null, null, definition.booleanType, null); } else if (expected.clazz == Boolean.class && internal) { - return new Cast(BOOLEAN_TYPE, BOOLEAN_TYPE, explicit, null, null, null, BOOLEAN_TYPE); + return new Cast(definition.booleanType, definition.booleanType, explicit, null, null, null, definition.booleanType); } } else if (actual.clazz == byte.class) { if (expected.dynamic) { - return new Cast(BYTE_OBJ_TYPE, DEF_TYPE, explicit, null, null, BYTE_TYPE, null); + return new Cast(definition.ByteType, definition.DefType, explicit, null, null, definition.byteType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(BYTE_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, BYTE_TYPE, null); + return new Cast(definition.ByteType, definition.ObjectType, explicit, null, null, definition.byteType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(BYTE_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, BYTE_TYPE, null); + return new Cast(definition.ByteType, definition.NumberType, explicit, null, null, definition.byteType, null); } else if (expected.clazz == short.class) { - return new Cast(BYTE_TYPE, SHORT_TYPE, explicit); + return new Cast(definition.byteType, definition.shortType, explicit); } else if (expected.clazz == char.class && explicit) { - return new Cast(BYTE_TYPE, CHAR_TYPE, true); + return new Cast(definition.byteType, definition.charType, true); } else if (expected.clazz == int.class) { - return new Cast(BYTE_TYPE, INT_TYPE, explicit); + return new Cast(definition.byteType, definition.intType, explicit); } else if (expected.clazz == long.class) { - return new Cast(BYTE_TYPE, LONG_TYPE, explicit); + return new Cast(definition.byteType, definition.longType, explicit); } else if (expected.clazz == float.class) { - return new Cast(BYTE_TYPE, FLOAT_TYPE, explicit); + return new Cast(definition.byteType, definition.floatType, explicit); } else if (expected.clazz == double.class) { - return new Cast(BYTE_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.byteType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && internal) { - return new Cast(BYTE_TYPE, BYTE_TYPE, explicit, null, null, null, BYTE_TYPE); + return new Cast(definition.byteType, definition.byteType, explicit, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && internal) { - return new Cast(BYTE_TYPE, SHORT_TYPE, explicit, null, null, null, SHORT_TYPE); + return new Cast(definition.byteType, definition.shortType, explicit, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(BYTE_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.byteType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && internal) { - return new Cast(BYTE_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE); + return new Cast(definition.byteType, definition.intType, explicit, null, null, null, definition.intType); } else if (expected.clazz == Long.class && internal) { - return new Cast(BYTE_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE); + return new Cast(definition.byteType, definition.longType, explicit, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(BYTE_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.byteType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(BYTE_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.byteType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == short.class) { if (expected.dynamic) { - return new Cast(SHORT_OBJ_TYPE, DEF_TYPE, explicit, null, null, SHORT_TYPE, null); + return new Cast(definition.ShortType, definition.DefType, explicit, null, null, definition.shortType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(SHORT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, SHORT_TYPE, null); + return new Cast(definition.ShortType, definition.ObjectType, explicit, null, null, definition.shortType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(SHORT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, SHORT_TYPE, null); + return new Cast(definition.ShortType, definition.NumberType, explicit, null, null, definition.shortType, null); } else if (expected.clazz == byte.class && explicit) { - return new Cast(SHORT_TYPE, BYTE_TYPE, true); + return new Cast(definition.shortType, definition.byteType, true); } else if (expected.clazz == char.class && explicit) { - return new Cast(SHORT_TYPE, CHAR_TYPE, true); + return new Cast(definition.shortType, definition.charType, true); } else if (expected.clazz == int.class) { - return new Cast(SHORT_TYPE, INT_TYPE, explicit); + return new Cast(definition.shortType, definition.intType, explicit); } else if (expected.clazz == long.class) { - return new Cast(SHORT_TYPE, LONG_TYPE, explicit); + return new Cast(definition.shortType, definition.longType, explicit); } else if (expected.clazz == float.class) { - return new Cast(SHORT_TYPE, FLOAT_TYPE, explicit); + return new Cast(definition.shortType, definition.floatType, explicit); } else if (expected.clazz == double.class) { - return new Cast(SHORT_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.shortType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(SHORT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.shortType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && internal) { - return new Cast(SHORT_TYPE, SHORT_TYPE, explicit, null, null, null, SHORT_TYPE); + return new Cast(definition.shortType, definition.shortType, explicit, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(SHORT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.shortType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && internal) { - return new Cast(SHORT_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE); + return new Cast(definition.shortType, definition.intType, explicit, null, null, null, definition.intType); } else if (expected.clazz == Long.class && internal) { - return new Cast(SHORT_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE); + return new Cast(definition.shortType, definition.longType, explicit, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(SHORT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.shortType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(SHORT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.shortType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == char.class) { if (expected.dynamic) { - return new Cast(CHAR_OBJ_TYPE, DEF_TYPE, explicit, null, null, CHAR_TYPE, null); + return new Cast(definition.CharacterType, definition.DefType, explicit, null, null, definition.charType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(CHAR_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, CHAR_TYPE, null); + return new Cast(definition.CharacterType, definition.ObjectType, explicit, null, null, definition.charType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(CHAR_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, CHAR_TYPE, null); + return new Cast(definition.CharacterType, definition.NumberType, explicit, null, null, definition.charType, null); } else if (expected.clazz == String.class) { - return new Cast(CHAR_TYPE, STRING_TYPE, explicit); + return new Cast(definition.charType, definition.StringType, explicit); } else if (expected.clazz == byte.class && explicit) { - return new Cast(CHAR_TYPE, BYTE_TYPE, true); + return new Cast(definition.charType, definition.byteType, true); } else if (expected.clazz == short.class && explicit) { - return new Cast(CHAR_TYPE, SHORT_TYPE, true); + return new Cast(definition.charType, definition.shortType, true); } else if (expected.clazz == int.class) { - return new Cast(CHAR_TYPE, INT_TYPE, explicit); + return new Cast(definition.charType, definition.intType, explicit); } else if (expected.clazz == long.class) { - return new Cast(CHAR_TYPE, LONG_TYPE, explicit); + return new Cast(definition.charType, definition.longType, explicit); } else if (expected.clazz == float.class) { - return new Cast(CHAR_TYPE, FLOAT_TYPE, explicit); + return new Cast(definition.charType, definition.floatType, explicit); } else if (expected.clazz == double.class) { - return new Cast(CHAR_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.charType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(CHAR_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.charType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && internal) { - return new Cast(CHAR_TYPE, SHORT_TYPE, explicit, null, null, null, SHORT_TYPE); + return new Cast(definition.charType, definition.shortType, explicit, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && internal) { - return new Cast(CHAR_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.charType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && internal) { - return new Cast(CHAR_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE); + return new Cast(definition.charType, definition.intType, explicit, null, null, null, definition.intType); } else if (expected.clazz == Long.class && internal) { - return new Cast(CHAR_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE); + return new Cast(definition.charType, definition.longType, explicit, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(CHAR_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.charType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(CHAR_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.charType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == int.class) { if (expected.dynamic) { - return new Cast(INT_OBJ_TYPE, DEF_TYPE, explicit, null, null, INT_TYPE, null); + return new Cast(definition.IntegerType, definition.DefType, explicit, null, null, definition.intType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(INT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, INT_TYPE, null); + return new Cast(definition.IntegerType, definition.ObjectType, explicit, null, null, definition.intType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(INT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, INT_TYPE, null); + return new Cast(definition.IntegerType, definition.NumberType, explicit, null, null, definition.intType, null); } else if (expected.clazz == byte.class && explicit) { - return new Cast(INT_TYPE, BYTE_TYPE, true); + return new Cast(definition.intType, definition.byteType, true); } else if (expected.clazz == char.class && explicit) { - return new Cast(INT_TYPE, CHAR_TYPE, true); + return new Cast(definition.intType, definition.charType, true); } else if (expected.clazz == short.class && explicit) { - return new Cast(INT_TYPE, SHORT_TYPE, true); + return new Cast(definition.intType, definition.shortType, true); } else if (expected.clazz == long.class) { - return new Cast(INT_TYPE, LONG_TYPE, explicit); + return new Cast(definition.intType, definition.longType, explicit); } else if (expected.clazz == float.class) { - return new Cast(INT_TYPE, FLOAT_TYPE, explicit); + return new Cast(definition.intType, definition.floatType, explicit); } else if (expected.clazz == double.class) { - return new Cast(INT_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.intType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(INT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.intType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && explicit && internal) { - return new Cast(INT_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE); + return new Cast(definition.intType, definition.shortType, true, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(INT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.intType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && internal) { - return new Cast(INT_TYPE, INT_TYPE, explicit, null, null, null, INT_TYPE); + return new Cast(definition.intType, definition.intType, explicit, null, null, null, definition.intType); } else if (expected.clazz == Long.class && internal) { - return new Cast(INT_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE); + return new Cast(definition.intType, definition.longType, explicit, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(INT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.intType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(INT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.intType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == long.class) { if (expected.dynamic) { - return new Cast(LONG_OBJ_TYPE, DEF_TYPE, explicit, null, null, LONG_TYPE, null); + return new Cast(definition.LongType, definition.DefType, explicit, null, null, definition.longType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(LONG_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, LONG_TYPE, null); + return new Cast(definition.LongType, definition.ObjectType, explicit, null, null, definition.longType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(LONG_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, LONG_TYPE, null); + return new Cast(definition.LongType, definition.NumberType, explicit, null, null, definition.longType, null); } else if (expected.clazz == byte.class && explicit) { - return new Cast(LONG_TYPE, BYTE_TYPE, true); + return new Cast(definition.longType, definition.byteType, true); } else if (expected.clazz == char.class && explicit) { - return new Cast(LONG_TYPE, CHAR_TYPE, true); + return new Cast(definition.longType, definition.charType, true); } else if (expected.clazz == short.class && explicit) { - return new Cast(LONG_TYPE, SHORT_TYPE, true); + return new Cast(definition.longType, definition.shortType, true); } else if (expected.clazz == int.class && explicit) { - return new Cast(LONG_TYPE, INT_TYPE, true); + return new Cast(definition.longType, definition.intType, true); } else if (expected.clazz == float.class) { - return new Cast(LONG_TYPE, FLOAT_TYPE, explicit); + return new Cast(definition.longType, definition.floatType, explicit); } else if (expected.clazz == double.class) { - return new Cast(LONG_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.longType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(LONG_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.longType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && explicit && internal) { - return new Cast(LONG_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE); + return new Cast(definition.longType, definition.shortType, true, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(LONG_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.longType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && explicit && internal) { - return new Cast(LONG_TYPE, INT_TYPE, true, null, null, null, INT_TYPE); + return new Cast(definition.longType, definition.intType, true, null, null, null, definition.intType); } else if (expected.clazz == Long.class && internal) { - return new Cast(LONG_TYPE, LONG_TYPE, explicit, null, null, null, LONG_TYPE); + return new Cast(definition.longType, definition.longType, explicit, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(LONG_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.longType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(LONG_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.longType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == float.class) { if (expected.dynamic) { - return new Cast(FLOAT_OBJ_TYPE, DEF_TYPE, explicit, null, null, FLOAT_TYPE, null); + return new Cast(definition.FloatType, definition.DefType, explicit, null, null, definition.floatType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(FLOAT_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, FLOAT_TYPE, null); + return new Cast(definition.FloatType, definition.ObjectType, explicit, null, null, definition.floatType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(FLOAT_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, FLOAT_TYPE, null); + return new Cast(definition.FloatType, definition.NumberType, explicit, null, null, definition.floatType, null); } else if (expected.clazz == byte.class && explicit) { - return new Cast(FLOAT_TYPE, BYTE_TYPE, true); + return new Cast(definition.floatType, definition.byteType, true); } else if (expected.clazz == char.class && explicit) { - return new Cast(FLOAT_TYPE, CHAR_TYPE, true); + return new Cast(definition.floatType, definition.charType, true); } else if (expected.clazz == short.class && explicit) { - return new Cast(FLOAT_TYPE, SHORT_TYPE, true); + return new Cast(definition.floatType, definition.shortType, true); } else if (expected.clazz == int.class && explicit) { - return new Cast(FLOAT_TYPE, INT_TYPE, true); + return new Cast(definition.floatType, definition.intType, true); } else if (expected.clazz == long.class && explicit) { - return new Cast(FLOAT_TYPE, LONG_TYPE, true); + return new Cast(definition.floatType, definition.longType, true); } else if (expected.clazz == double.class) { - return new Cast(FLOAT_TYPE, DOUBLE_TYPE, explicit); + return new Cast(definition.floatType, definition.doubleType, explicit); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(FLOAT_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.floatType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && explicit && internal) { - return new Cast(FLOAT_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE); + return new Cast(definition.floatType, definition.shortType, true, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(FLOAT_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.floatType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && explicit && internal) { - return new Cast(FLOAT_TYPE, INT_TYPE, true, null, null, null, INT_TYPE); + return new Cast(definition.floatType, definition.intType, true, null, null, null, definition.intType); } else if (expected.clazz == Long.class && explicit && internal) { - return new Cast(FLOAT_TYPE, LONG_TYPE, true, null, null, null, LONG_TYPE); + return new Cast(definition.floatType, definition.longType, true, null, null, null, definition.longType); } else if (expected.clazz == Float.class && internal) { - return new Cast(FLOAT_TYPE, FLOAT_TYPE, explicit, null, null, null, FLOAT_TYPE); + return new Cast(definition.floatType, definition.floatType, explicit, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(FLOAT_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.floatType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == double.class) { if (expected.dynamic) { - return new Cast(DOUBLE_OBJ_TYPE, DEF_TYPE, explicit, null, null, DOUBLE_TYPE, null); + return new Cast(definition.DoubleType, definition.DefType, explicit, null, null, definition.doubleType, null); } else if (expected.clazz == Object.class && internal) { - return new Cast(DOUBLE_OBJ_TYPE, OBJECT_TYPE, explicit, null, null, DOUBLE_TYPE, null); + return new Cast(definition.DoubleType, definition.ObjectType, explicit, null, null, definition.doubleType, null); } else if (expected.clazz == Number.class && internal) { - return new Cast(DOUBLE_OBJ_TYPE, NUMBER_TYPE, explicit, null, null, DOUBLE_TYPE, null); + return new Cast(definition.DoubleType, definition.NumberType, explicit, null, null, definition.doubleType, null); } else if (expected.clazz == byte.class && explicit) { - return new Cast(DOUBLE_TYPE, BYTE_TYPE, true); + return new Cast(definition.doubleType, definition.byteType, true); } else if (expected.clazz == char.class && explicit) { - return new Cast(DOUBLE_TYPE, CHAR_TYPE, true); + return new Cast(definition.doubleType, definition.charType, true); } else if (expected.clazz == short.class && explicit) { - return new Cast(DOUBLE_TYPE, SHORT_TYPE, true); + return new Cast(definition.doubleType, definition.shortType, true); } else if (expected.clazz == int.class && explicit) { - return new Cast(DOUBLE_TYPE, INT_TYPE, true); + return new Cast(definition.doubleType, definition.intType, true); } else if (expected.clazz == long.class && explicit) { - return new Cast(DOUBLE_TYPE, LONG_TYPE, true); + return new Cast(definition.doubleType, definition.longType, true); } else if (expected.clazz == float.class && explicit) { - return new Cast(DOUBLE_TYPE, FLOAT_TYPE, true); + return new Cast(definition.doubleType, definition.floatType, true); } else if (expected.clazz == Byte.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, BYTE_TYPE, true, null, null, null, BYTE_TYPE); + return new Cast(definition.doubleType, definition.byteType, true, null, null, null, definition.byteType); } else if (expected.clazz == Short.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, SHORT_TYPE, true, null, null, null, SHORT_TYPE); + return new Cast(definition.doubleType, definition.shortType, true, null, null, null, definition.shortType); } else if (expected.clazz == Character.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, CHAR_TYPE, true, null, null, null, CHAR_TYPE); + return new Cast(definition.doubleType, definition.charType, true, null, null, null, definition.charType); } else if (expected.clazz == Integer.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, INT_TYPE, true, null, null, null, INT_TYPE); + return new Cast(definition.doubleType, definition.intType, true, null, null, null, definition.intType); } else if (expected.clazz == Long.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, LONG_TYPE, true, null, null, null, LONG_TYPE); + return new Cast(definition.doubleType, definition.longType, true, null, null, null, definition.longType); } else if (expected.clazz == Float.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, FLOAT_TYPE, true, null, null, null, FLOAT_TYPE); + return new Cast(definition.doubleType, definition.floatType, true, null, null, null, definition.floatType); } else if (expected.clazz == Double.class && internal) { - return new Cast(DOUBLE_TYPE, DOUBLE_TYPE, explicit, null, null, null, DOUBLE_TYPE); + return new Cast(definition.doubleType, definition.doubleType, explicit, null, null, null, definition.doubleType); } } else if (actual.clazz == Boolean.class) { if (expected.clazz == boolean.class && internal) { - return new Cast(BOOLEAN_TYPE, BOOLEAN_TYPE, explicit, BOOLEAN_TYPE, null, null, null); + return new Cast(definition.booleanType, definition.booleanType, explicit, definition.booleanType, null, null, null); } } else if (actual.clazz == Byte.class) { if (expected.clazz == byte.class && internal) { - return new Cast(BYTE_TYPE, BYTE_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.byteType, explicit, definition.byteType, null, null, null); } else if (expected.clazz == short.class && internal) { - return new Cast(BYTE_TYPE, SHORT_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.shortType, explicit, definition.byteType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(BYTE_TYPE, CHAR_TYPE, true, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.charType, true, definition.byteType, null, null, null); } else if (expected.clazz == int.class && internal) { - return new Cast(BYTE_TYPE, INT_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.intType, explicit, definition.byteType, null, null, null); } else if (expected.clazz == long.class && internal) { - return new Cast(BYTE_TYPE, LONG_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.longType, explicit, definition.byteType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(BYTE_TYPE, FLOAT_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.floatType, explicit, definition.byteType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(BYTE_TYPE, DOUBLE_TYPE, explicit, BYTE_TYPE, null, null, null); + return new Cast(definition.byteType, definition.doubleType, explicit, definition.byteType, null, null, null); } } else if (actual.clazz == Short.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(SHORT_TYPE, BYTE_TYPE, true, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.byteType, true, definition.shortType, null, null, null); } else if (expected.clazz == short.class && internal) { - return new Cast(SHORT_TYPE, SHORT_TYPE, explicit, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.shortType, explicit, definition.shortType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(SHORT_TYPE, CHAR_TYPE, true, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.charType, true, definition.shortType, null, null, null); } else if (expected.clazz == int.class && internal) { - return new Cast(SHORT_TYPE, INT_TYPE, explicit, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.intType, explicit, definition.shortType, null, null, null); } else if (expected.clazz == long.class && internal) { - return new Cast(SHORT_TYPE, LONG_TYPE, explicit, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.longType, explicit, definition.shortType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(SHORT_TYPE, FLOAT_TYPE, explicit, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.floatType, explicit, definition.shortType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(SHORT_TYPE, DOUBLE_TYPE, explicit, SHORT_TYPE, null, null, null); + return new Cast(definition.shortType, definition.doubleType, explicit, definition.shortType, null, null, null); } } else if (actual.clazz == Character.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(CHAR_TYPE, BYTE_TYPE, true, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.byteType, true, definition.charType, null, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(CHAR_TYPE, SHORT_TYPE, true, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.shortType, true, definition.charType, null, null, null); } else if (expected.clazz == char.class && internal) { - return new Cast(CHAR_TYPE, CHAR_TYPE, explicit, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.charType, explicit, definition.charType, null, null, null); } else if (expected.clazz == int.class && internal) { - return new Cast(CHAR_TYPE, INT_TYPE, explicit, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.intType, explicit, definition.charType, null, null, null); } else if (expected.clazz == long.class && internal) { - return new Cast(CHAR_TYPE, LONG_TYPE, explicit, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.longType, explicit, definition.charType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(CHAR_TYPE, FLOAT_TYPE, explicit, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.floatType, explicit, definition.charType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(CHAR_TYPE, DOUBLE_TYPE, explicit, CHAR_TYPE, null, null, null); + return new Cast(definition.charType, definition.doubleType, explicit, definition.charType, null, null, null); } } else if (actual.clazz == Integer.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(INT_TYPE, BYTE_TYPE, true, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.byteType, true, definition.intType, null, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(INT_TYPE, SHORT_TYPE, true, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.shortType, true, definition.intType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(INT_TYPE, CHAR_TYPE, true, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.charType, true, definition.intType, null, null, null); } else if (expected.clazz == int.class && internal) { - return new Cast(INT_TYPE, INT_TYPE, explicit, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.intType, explicit, definition.intType, null, null, null); } else if (expected.clazz == long.class && internal) { - return new Cast(INT_TYPE, LONG_TYPE, explicit, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.longType, explicit, definition.intType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(INT_TYPE, FLOAT_TYPE, explicit, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.floatType, explicit, definition.intType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(INT_TYPE, DOUBLE_TYPE, explicit, INT_TYPE, null, null, null); + return new Cast(definition.intType, definition.doubleType, explicit, definition.intType, null, null, null); } } else if (actual.clazz == Long.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(LONG_TYPE, BYTE_TYPE, true, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.byteType, true, definition.longType, null, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(LONG_TYPE, SHORT_TYPE, true, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.shortType, true, definition.longType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(LONG_TYPE, CHAR_TYPE, true, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.charType, true, definition.longType, null, null, null); } else if (expected.clazz == int.class && explicit && internal) { - return new Cast(LONG_TYPE, INT_TYPE, true, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.intType, true, definition.longType, null, null, null); } else if (expected.clazz == long.class && internal) { - return new Cast(LONG_TYPE, LONG_TYPE, explicit, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.longType, explicit, definition.longType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(LONG_TYPE, FLOAT_TYPE, explicit, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.floatType, explicit, definition.longType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(LONG_TYPE, DOUBLE_TYPE, explicit, LONG_TYPE, null, null, null); + return new Cast(definition.longType, definition.doubleType, explicit, definition.longType, null, null, null); } } else if (actual.clazz == Float.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(FLOAT_TYPE, BYTE_TYPE, true, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.byteType, true, definition.floatType, null, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(FLOAT_TYPE, SHORT_TYPE, true, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.shortType, true, definition.floatType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(FLOAT_TYPE, CHAR_TYPE, true, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.charType, true, definition.floatType, null, null, null); } else if (expected.clazz == int.class && explicit && internal) { - return new Cast(FLOAT_TYPE, INT_TYPE, true, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.intType, true, definition.floatType, null, null, null); } else if (expected.clazz == long.class && explicit && internal) { - return new Cast(FLOAT_TYPE, LONG_TYPE, true, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.longType, true, definition.floatType, null, null, null); } else if (expected.clazz == float.class && internal) { - return new Cast(FLOAT_TYPE, FLOAT_TYPE, explicit, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.floatType, explicit, definition.floatType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(FLOAT_TYPE, DOUBLE_TYPE, explicit, FLOAT_TYPE, null, null, null); + return new Cast(definition.floatType, definition.doubleType, explicit, definition.floatType, null, null, null); } } else if (actual.clazz == Double.class) { if (expected.clazz == byte.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, BYTE_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.byteType, true, definition.doubleType, null, null, null); } else if (expected.clazz == short.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, SHORT_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.shortType, true, definition.doubleType, null, null, null); } else if (expected.clazz == char.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, CHAR_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.charType, true, definition.doubleType, null, null, null); } else if (expected.clazz == int.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, INT_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.intType, true, definition.doubleType, null, null, null); } else if (expected.clazz == long.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, LONG_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.longType, true, definition.doubleType, null, null, null); } else if (expected.clazz == float.class && explicit && internal) { - return new Cast(DOUBLE_TYPE, FLOAT_TYPE, true, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.floatType, true, definition.doubleType, null, null, null); } else if (expected.clazz == double.class && internal) { - return new Cast(DOUBLE_TYPE, DOUBLE_TYPE, explicit, DOUBLE_TYPE, null, null, null); + return new Cast(definition.doubleType, definition.doubleType, explicit, definition.doubleType, null, null, null); } } @@ -489,7 +474,7 @@ public final class AnalyzerCaster { } } - public static Object constCast(Location location, final Object constant, final Cast cast) { + public Object constCast(Location location, final Object constant, final Cast cast) { Class fsort = cast.from.clazz; Class tsort = cast.to.clazz; @@ -525,98 +510,98 @@ public final class AnalyzerCaster { } } - public static Type promoteNumeric(Type from, boolean decimal) { + public Type promoteNumeric(Type from, boolean decimal) { Class sort = from.clazz; if (from.dynamic) { - return DEF_TYPE; + return definition.DefType; } else if ((sort == double.class) && decimal) { - return DOUBLE_TYPE; + return definition.doubleType; } else if ((sort == float.class) && decimal) { - return FLOAT_TYPE; + return definition.floatType; } else if (sort == long.class) { - return LONG_TYPE; + return definition.longType; } else if (sort == int.class || sort == char.class || sort == short.class || sort == byte.class) { - return INT_TYPE; + return definition.intType; } return null; } - public static Type promoteNumeric(Type from0, Type from1, boolean decimal) { + public Type promoteNumeric(Type from0, Type from1, boolean decimal) { Class sort0 = from0.clazz; Class sort1 = from1.clazz; if (from0.dynamic || from1.dynamic) { - return DEF_TYPE; + return definition.DefType; } if (decimal) { if (sort0 == double.class || sort1 == double.class) { - return DOUBLE_TYPE; + return definition.doubleType; } else if (sort0 == float.class || sort1 == float.class) { - return FLOAT_TYPE; + return definition.floatType; } } if (sort0 == long.class || sort1 == long.class) { - return LONG_TYPE; + return definition.longType; } else if (sort0 == int.class || sort1 == int.class || sort0 == char.class || sort1 == char.class || sort0 == short.class || sort1 == short.class || sort0 == byte.class || sort1 == byte.class) { - return INT_TYPE; + return definition.intType; } return null; } - public static Type promoteAdd(Type from0, Type from1) { + public Type promoteAdd(Type from0, Type from1) { Class sort0 = from0.clazz; Class sort1 = from1.clazz; if (sort0 == String.class || sort1 == String.class) { - return STRING_TYPE; + return definition.StringType; } return promoteNumeric(from0, from1, true); } - public static Type promoteXor(Type from0, Type from1) { + public Type promoteXor(Type from0, Type from1) { Class sort0 = from0.clazz; Class sort1 = from1.clazz; if (from0.dynamic || from1.dynamic) { - return DEF_TYPE; + return definition.DefType; } if (sort0 == boolean.class || sort1 == boolean.class) { - return BOOLEAN_TYPE; + return definition.booleanType; } return promoteNumeric(from0, from1, false); } - public static Type promoteEquality(Type from0, Type from1) { + public Type promoteEquality(Type from0, Type from1) { Class sort0 = from0.clazz; Class sort1 = from1.clazz; if (from0.dynamic || from1.dynamic) { - return DEF_TYPE; + return definition.DefType; } if (sort0.isPrimitive() && sort1.isPrimitive()) { if (sort0 == boolean.class && sort1 == boolean.class) { - return BOOLEAN_TYPE; + return definition.booleanType; } return promoteNumeric(from0, from1, true); } - return OBJECT_TYPE; + return definition.ObjectType; } - public static Type promoteConditional(Type from0, Type from1, Object const0, Object const1) { + public Type promoteConditional(Type from0, Type from1, Object const0, Object const1) { if (from0.equals(from1)) { return from0; } @@ -625,46 +610,46 @@ public final class AnalyzerCaster { Class sort1 = from1.clazz; if (from0.dynamic || from1.dynamic) { - return DEF_TYPE; + return definition.DefType; } if (sort0.isPrimitive() && sort1.isPrimitive()) { if (sort0 == boolean.class && sort1 == boolean.class) { - return BOOLEAN_TYPE; + return definition.booleanType; } if (sort0 == double.class || sort1 == double.class) { - return DOUBLE_TYPE; + return definition.doubleType; } else if (sort0 == float.class || sort1 == float.class) { - return FLOAT_TYPE; + return definition.floatType; } else if (sort0 == long.class || sort1 == long.class) { - return LONG_TYPE; + return definition.longType; } else { if (sort0 == byte.class) { if (sort1 == byte.class) { - return BYTE_TYPE; + return definition.byteType; } else if (sort1 == short.class) { if (const1 != null) { final short constant = (short)const1; if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return SHORT_TYPE; + return definition.shortType; } else if (sort1 == char.class) { - return INT_TYPE; + return definition.intType; } else if (sort1 == int.class) { if (const1 != null) { final int constant = (int)const1; if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return INT_TYPE; + return definition.intType; } } else if (sort0 == short.class) { if (sort1 == byte.class) { @@ -672,43 +657,43 @@ public final class AnalyzerCaster { final short constant = (short)const0; if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return SHORT_TYPE; + return definition.shortType; } else if (sort1 == short.class) { - return SHORT_TYPE; + return definition.shortType; } else if (sort1 == char.class) { - return INT_TYPE; + return definition.intType; } else if (sort1 == int.class) { if (const1 != null) { final int constant = (int)const1; if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) { - return SHORT_TYPE; + return definition.shortType; } } - return INT_TYPE; + return definition.intType; } } else if (sort0 == char.class) { if (sort1 == byte.class) { - return INT_TYPE; + return definition.intType; } else if (sort1 == short.class) { - return INT_TYPE; + return definition.intType; } else if (sort1 == char.class) { - return CHAR_TYPE; + return definition.charType; } else if (sort1 == int.class) { if (const1 != null) { final int constant = (int)const1; if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return INT_TYPE; + return definition.intType; } } else if (sort0 == int.class) { if (sort1 == byte.class) { @@ -716,33 +701,33 @@ public final class AnalyzerCaster { final int constant = (int)const0; if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return INT_TYPE; + return definition.intType; } else if (sort1 == short.class) { if (const0 != null) { final int constant = (int)const0; if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return INT_TYPE; + return definition.intType; } else if (sort1 == char.class) { if (const0 != null) { final int constant = (int)const0; if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) { - return BYTE_TYPE; + return definition.byteType; } } - return INT_TYPE; + return definition.intType; } else if (sort1 == int.class) { - return INT_TYPE; + return definition.intType; } } } @@ -752,8 +737,6 @@ public final class AnalyzerCaster { // TODO: to calculate the highest upper bound for the two types and return that. // TODO: However, for now we just return objectType that may require an extra cast. - return OBJECT_TYPE; + return definition.ObjectType; } - - private AnalyzerCaster() {} } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java index 6589c544af7..e5bfb82c731 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java @@ -19,24 +19,19 @@ package org.elasticsearch.painless; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.SetOnce; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; -import java.time.LocalDate; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.PrimitiveIterator; -import java.util.Spliterator; import java.util.Stack; /** @@ -65,37 +60,37 @@ public final class Definition { /** * Whitelist that is "built in" to Painless and required by all scripts. */ - public static final Definition BUILTINS = new Definition( + public static final Definition DEFINITION = new Definition( Collections.singletonList(WhitelistLoader.loadFromResourceFiles(Definition.class, DEFINITION_FILES))); /** Some native types as constants: */ - public static final Type VOID_TYPE = BUILTINS.getType("void"); - public static final Type BOOLEAN_TYPE = BUILTINS.getType("boolean"); - public static final Type BOOLEAN_OBJ_TYPE = BUILTINS.getType("Boolean"); - public static final Type BYTE_TYPE = BUILTINS.getType("byte"); - public static final Type BYTE_OBJ_TYPE = BUILTINS.getType("Byte"); - public static final Type SHORT_TYPE = BUILTINS.getType("short"); - public static final Type SHORT_OBJ_TYPE = BUILTINS.getType("Short"); - public static final Type INT_TYPE = BUILTINS.getType("int"); - public static final Type INT_OBJ_TYPE = BUILTINS.getType("Integer"); - public static final Type LONG_TYPE = BUILTINS.getType("long"); - public static final Type LONG_OBJ_TYPE = BUILTINS.getType("Long"); - public static final Type FLOAT_TYPE = BUILTINS.getType("float"); - public static final Type FLOAT_OBJ_TYPE = BUILTINS.getType("Float"); - public static final Type DOUBLE_TYPE = BUILTINS.getType("double"); - public static final Type DOUBLE_OBJ_TYPE = BUILTINS.getType("Double"); - public static final Type CHAR_TYPE = BUILTINS.getType("char"); - public static final Type CHAR_OBJ_TYPE = BUILTINS.getType("Character"); - public static final Type OBJECT_TYPE = BUILTINS.getType("Object"); - public static final Type DEF_TYPE = BUILTINS.getType("def"); - public static final Type NUMBER_TYPE = BUILTINS.getType("Number"); - public static final Type STRING_TYPE = BUILTINS.getType("String"); - public static final Type EXCEPTION_TYPE = BUILTINS.getType("Exception"); - public static final Type PATTERN_TYPE = BUILTINS.getType("Pattern"); - public static final Type MATCHER_TYPE = BUILTINS.getType("Matcher"); - public static final Type ITERATOR_TYPE = BUILTINS.getType("Iterator"); - public static final Type ARRAY_LIST_TYPE = BUILTINS.getType("ArrayList"); - public static final Type HASH_MAP_TYPE = BUILTINS.getType("HashMap"); + public final Type voidType; + public final Type booleanType; + public final Type BooleanType; + public final Type byteType; + public final Type ByteType; + public final Type shortType; + public final Type ShortType; + public final Type intType; + public final Type IntegerType; + public final Type longType; + public final Type LongType; + public final Type floatType; + public final Type FloatType; + public final Type doubleType; + public final Type DoubleType; + public final Type charType; + public final Type CharacterType; + public final Type ObjectType; + public final Type DefType; + public final Type NumberType; + public final Type StringType; + public final Type ExceptionType; + public final Type PatternType; + public final Type MatcherType; + public final Type IteratorType; + public final Type ArrayListType; + public final Type HashMapType; public static final class Type { public final String name; @@ -438,58 +433,58 @@ public final class Definition { /** Returns whether or not a non-array type exists. */ public boolean isSimpleType(final String name) { - return BUILTINS.structsMap.containsKey(name); + return structsMap.containsKey(name); } /** Gets the type given by its name */ public Type getType(final String name) { - return BUILTINS.getTypeInternal(name); + return getTypeInternal(name); } /** Creates an array type from the given Struct. */ public Type getType(final Struct struct, final int dimensions) { - return BUILTINS.getTypeInternal(struct, dimensions); + return getTypeInternal(struct, dimensions); } - public static Type getBoxedType(Type unboxed) { + public Type getBoxedType(Type unboxed) { if (unboxed.clazz == boolean.class) { - return BOOLEAN_OBJ_TYPE; + return BooleanType; } else if (unboxed.clazz == byte.class) { - return BYTE_OBJ_TYPE; + return ByteType; } else if (unboxed.clazz == short.class) { - return SHORT_OBJ_TYPE; + return ShortType; } else if (unboxed.clazz == char.class) { - return CHAR_OBJ_TYPE; + return CharacterType; } else if (unboxed.clazz == int.class) { - return INT_OBJ_TYPE; + return IntegerType; } else if (unboxed.clazz == long.class) { - return LONG_OBJ_TYPE; + return LongType; } else if (unboxed.clazz == float.class) { - return FLOAT_OBJ_TYPE; + return FloatType; } else if (unboxed.clazz == double.class) { - return DOUBLE_OBJ_TYPE; + return DoubleType; } return unboxed; } - public static Type getUnboxedType(Type boxed) { + public Type getUnboxedType(Type boxed) { if (boxed.clazz == Boolean.class) { - return BOOLEAN_TYPE; + return booleanType; } else if (boxed.clazz == Byte.class) { - return BYTE_TYPE; + return byteType; } else if (boxed.clazz == Short.class) { - return SHORT_TYPE; + return shortType; } else if (boxed.clazz == Character.class) { - return CHAR_TYPE; + return charType; } else if (boxed.clazz == Integer.class) { - return INT_TYPE; + return intType; } else if (boxed.clazz == Long.class) { - return LONG_TYPE; + return longType; } else if (boxed.clazz == Float.class) { - return FLOAT_TYPE; + return floatType; } else if (boxed.clazz == Double.class) { - return DOUBLE_TYPE; + return doubleType; } return boxed; @@ -508,12 +503,12 @@ public final class Definition { } public RuntimeClass getRuntimeClass(Class clazz) { - return BUILTINS.runtimeMap.get(clazz); + return runtimeMap.get(clazz); } /** Collection of all simple types. Used by {@code PainlessDocGenerator} to generate an API reference. */ - static Collection allSimpleTypes() { - return BUILTINS.simpleTypesMap.values(); + Collection allSimpleTypes() { + return simpleTypesMap.values(); } // INTERNAL IMPLEMENTATION: @@ -522,6 +517,8 @@ public final class Definition { private final Map structsMap; private final Map simpleTypesMap; + public AnalyzerCaster caster; + private Definition(List whitelists) { structsMap = new HashMap<>(); simpleTypesMap = new HashMap<>(); @@ -648,6 +645,36 @@ public final class Definition { for (final Map.Entry entry : structsMap.entrySet()) { entry.setValue(entry.getValue().freeze()); } + + voidType = getType("void"); + booleanType = getType("boolean"); + BooleanType = getType("Boolean"); + byteType = getType("byte"); + ByteType = getType("Byte"); + shortType = getType("short"); + ShortType = getType("Short"); + intType = getType("int"); + IntegerType = getType("Integer"); + longType = getType("long"); + LongType = getType("Long"); + floatType = getType("float"); + FloatType = getType("Float"); + doubleType = getType("double"); + DoubleType = getType("Double"); + charType = getType("char"); + CharacterType = getType("Character"); + ObjectType = getType("Object"); + DefType = getType("def"); + NumberType = getType("Number"); + StringType = getType("String"); + ExceptionType = getType("Exception"); + PatternType = getType("Pattern"); + MatcherType = getType("Matcher"); + IteratorType = getType("Iterator"); + ArrayListType = getType("ArrayList"); + HashMapType = getType("HashMap"); + + caster = new AnalyzerCaster(this); } private void addStruct(ClassLoader whitelistClassLoader, Whitelist.Struct whitelistStruct) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java index b83a16df3ae..9f769d48413 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java @@ -72,7 +72,7 @@ public final class Locals { } // Loop counter to catch infinite loops. Internal use only. if (maxLoopCounter > 0) { - locals.defineVariable(null, Definition.INT_TYPE, LOOP, true); + locals.defineVariable(null, locals.getDefinition().intType, LOOP, true); } return locals; } @@ -85,7 +85,7 @@ public final class Locals { } // Loop counter to catch infinite loops. Internal use only. if (maxLoopCounter > 0) { - locals.defineVariable(null, Definition.INT_TYPE, LOOP, true); + locals.defineVariable(null, locals.getDefinition().intType, LOOP, true); } return locals; } @@ -104,7 +104,7 @@ public final class Locals { // Loop counter to catch infinite loops. Internal use only. if (maxLoopCounter > 0) { - locals.defineVariable(null, Definition.INT_TYPE, LOOP, true); + locals.defineVariable(null, locals.getDefinition().intType, LOOP, true); } return locals; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index faf8521616f..efdd36172d4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -38,7 +38,7 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin { // force to parse our definition at startup (not on the user's first script) static { - Definition.VOID_TYPE.hashCode(); + Definition.DEFINITION.hashCode(); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 39f5c48b65e..5299adb1dc8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -101,9 +101,9 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr for (ScriptContext context : contexts) { if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) { - contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, Definition.BUILTINS)); + contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, Definition.DEFINITION)); } else { - contextsToCompilers.put(context, new Compiler(context.instanceClazz, Definition.BUILTINS)); + contextsToCompilers.put(context, new Compiler(context.instanceClazz, Definition.DEFINITION)); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java index abf834c8acb..e52aaf25969 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java @@ -191,7 +191,7 @@ public class ScriptClassInfo { } Definition.Struct struct; if (componentType.equals(Object.class)) { - struct = Definition.DEF_TYPE.struct; + struct = definition.DefType.struct; } else { Definition.RuntimeClass runtimeClass = definition.getRuntimeClass(componentType); if (runtimeClass == null) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 60eff7754e8..9150e2609b7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -137,7 +137,7 @@ public final class WriterConstants { public static final Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); public static final Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); public static final Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); - public static final Type DEF_ARRAY_LENGTH_METHOD_TYPE = Type.getMethodType(Type.INT_TYPE, Definition.DEF_TYPE.type); + public static final Type DEF_ARRAY_LENGTH_METHOD_TYPE = Type.getMethodType(Type.INT_TYPE, Type.getType(Object.class)); /** invokedynamic bootstrap for lambda expression/method references */ public static final MethodType LAMBDA_BOOTSTRAP_TYPE = diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java index fd32c59b4ff..734089c384a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java @@ -11,7 +11,7 @@ import org.antlr.v4.runtime.misc.*; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) abstract class PainlessLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java index 619c582d04a..528a8a3d851 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java @@ -11,7 +11,7 @@ import java.util.ArrayList; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) class PainlessParser extends Parser { - static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = @@ -579,6 +579,7 @@ class PainlessParser extends Parser { try { int _alt; setState(185); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: _localctx = new IfContext(_localctx); @@ -595,6 +596,7 @@ class PainlessParser extends Parser { setState(103); trailer(); setState(107); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) { case 1: { @@ -1127,6 +1129,7 @@ class PainlessParser extends Parser { enterRule(_localctx, 14, RULE_initializer); try { setState(204); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -1653,6 +1656,7 @@ class PainlessParser extends Parser { _prevctx = _localctx; { setState(290); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) { case 1: { @@ -1979,6 +1983,7 @@ class PainlessParser extends Parser { int _la; try { setState(308); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: _localctx = new PreContext(_localctx); @@ -2126,6 +2131,7 @@ class PainlessParser extends Parser { try { int _alt; setState(326); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { case 1: _localctx = new DynamicContext(_localctx); @@ -2342,6 +2348,7 @@ class PainlessParser extends Parser { int _la; try { setState(346); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: _localctx = new PrecedenceContext(_localctx); @@ -2493,6 +2500,7 @@ class PainlessParser extends Parser { enterRule(_localctx, 36, RULE_postfix); try { setState(351); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -2551,6 +2559,7 @@ class PainlessParser extends Parser { enterRule(_localctx, 38, RULE_postdot); try { setState(355); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -2803,6 +2812,7 @@ class PainlessParser extends Parser { try { int _alt; setState(412); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { case 1: _localctx = new NewstandardarrayContext(_localctx); @@ -2837,6 +2847,7 @@ class PainlessParser extends Parser { _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); setState(385); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { @@ -2974,6 +2985,7 @@ class PainlessParser extends Parser { int _la; try { setState(427); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -3055,6 +3067,7 @@ class PainlessParser extends Parser { int _la; try { setState(443); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -3252,6 +3265,7 @@ class PainlessParser extends Parser { enterRule(_localctx, 56, RULE_argument); try { setState(465); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); @@ -3533,6 +3547,7 @@ class PainlessParser extends Parser { enterRule(_localctx, 62, RULE_funcref); try { setState(505); + _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: _localctx = new ClassfuncrefContext(_localctx); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java index 6cfd859a2b4..2ca0b265430 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java @@ -119,7 +119,7 @@ public abstract class AExpression extends ANode { * @return The new child node for the parent node calling this method. */ AExpression cast(Locals locals) { - Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal); + Cast cast = locals.getDefinition().caster.getLegalCast(location, actual, expected, explicit, internal); if (cast == null) { if (constant == null || this instanceof EConstant) { @@ -167,7 +167,7 @@ public abstract class AExpression extends ANode { // from this node because the output data for the EConstant // will already be the same. - constant = AnalyzerCaster.constCast(location, constant, cast); + constant = locals.getDefinition().caster.constCast(location, constant, cast); EConstant econstant = new EConstant(location, constant); econstant.analyze(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java index 70b85205e24..84c6145ac0c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java @@ -19,7 +19,6 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Cast; @@ -51,6 +50,7 @@ public final class EAssignment extends AExpression { private Type shiftDistance; // for shifts, the RHS is promoted independently private Cast there = null; private Cast back = null; + private Type DefType = null; public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean pre, boolean post, Operation operation) { super(location); @@ -80,6 +80,8 @@ public final class EAssignment extends AExpression { } else { throw new IllegalStateException("Illegal tree structure."); } + + DefType = locals.getDefinition().DefType; } private void analyzeLHS(Locals locals) { @@ -138,33 +140,33 @@ public final class EAssignment extends AExpression { boolean shift = false; if (operation == Operation.MUL) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, rhs.actual, true); } else if (operation == Operation.DIV) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, rhs.actual, true); } else if (operation == Operation.REM) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, rhs.actual, true); } else if (operation == Operation.ADD) { - promote = AnalyzerCaster.promoteAdd(lhs.actual, rhs.actual); + promote = locals.getDefinition().caster.promoteAdd(lhs.actual, rhs.actual); } else if (operation == Operation.SUB) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, rhs.actual, true); } else if (operation == Operation.LSH) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, false); - shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, false); + shiftDistance = locals.getDefinition().caster.promoteNumeric(rhs.actual, false); shift = true; } else if (operation == Operation.RSH) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, false); - shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, false); + shiftDistance = locals.getDefinition().caster.promoteNumeric(rhs.actual, false); shift = true; } else if (operation == Operation.USH) { - promote = AnalyzerCaster.promoteNumeric(lhs.actual, false); - shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false); + promote = locals.getDefinition().caster.promoteNumeric(lhs.actual, false); + shiftDistance = locals.getDefinition().caster.promoteNumeric(rhs.actual, false); shift = true; } else if (operation == Operation.BWAND) { - promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual); + promote = locals.getDefinition().caster.promoteXor(lhs.actual, rhs.actual); } else if (operation == Operation.XOR) { - promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual); + promote = locals.getDefinition().caster.promoteXor(lhs.actual, rhs.actual); } else if (operation == Operation.BWOR) { - promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual); + promote = locals.getDefinition().caster.promoteXor(lhs.actual, rhs.actual); } else { throw createError(new IllegalStateException("Illegal tree structure.")); } @@ -187,7 +189,7 @@ public final class EAssignment extends AExpression { // shifts are promoted independently, but for the def type, we need object. rhs.expected = promote; } else if (shiftDistance.clazz == long.class) { - rhs.expected = Definition.INT_TYPE; + rhs.expected = locals.getDefinition().intType; rhs.explicit = true; } else { rhs.expected = shiftDistance; @@ -198,11 +200,11 @@ public final class EAssignment extends AExpression { rhs = rhs.cast(locals); - there = AnalyzerCaster.getLegalCast(location, lhs.actual, promote, false, false); - back = AnalyzerCaster.getLegalCast(location, promote, lhs.actual, true, false); + there = locals.getDefinition().caster.getLegalCast(location, lhs.actual, promote, false, false); + back = locals.getDefinition().caster.getLegalCast(location, promote, lhs.actual, true, false); this.statement = true; - this.actual = read ? lhs.actual : Definition.VOID_TYPE; + this.actual = read ? lhs.actual : locals.getDefinition().voidType; } private void analyzeSimple(Locals locals) { @@ -222,7 +224,7 @@ public final class EAssignment extends AExpression { rhs = rhs.cast(locals); this.statement = true; - this.actual = read ? lhs.actual : Definition.VOID_TYPE; + this.actual = read ? lhs.actual : locals.getDefinition().voidType; } /** @@ -295,8 +297,8 @@ public final class EAssignment extends AExpression { // its tricky here as there are possibly explicit casts, too. // write the operation instruction for compound assignment if (promote.dynamic) { - writer.writeDynamicBinaryInstruction(location, promote, - Definition.DEF_TYPE, Definition.DEF_TYPE, operation, DefBootstrap.OPERATOR_COMPOUND_ASSIGNMENT); + writer.writeDynamicBinaryInstruction( + location, promote, DefType, DefType, operation, DefBootstrap.OPERATOR_COMPOUND_ASSIGNMENT); } else { writer.writeBinaryInstruction(location, promote, operation); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java index 6cd0fc8a6dc..df92d72a3c0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java @@ -19,7 +19,6 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Type; @@ -32,6 +31,8 @@ import org.elasticsearch.painless.WriterConstants; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Represents a binary math expression. @@ -100,7 +101,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply multiply [*] to types " + @@ -144,7 +145,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply divide [/] to types " + @@ -193,7 +194,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply remainder [%] to types " + @@ -242,7 +243,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteAdd(left.actual, right.actual); + promote = variables.getDefinition().caster.promoteAdd(left.actual, right.actual); if (promote == null) { throw createError(new ClassCastException("Cannot apply add [+] to types " + @@ -302,7 +303,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply subtract [-] to types " + @@ -347,22 +348,22 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - left.expected = Definition.STRING_TYPE; - right.expected = Definition.PATTERN_TYPE; + left.expected = variables.getDefinition().StringType; + right.expected = variables.getDefinition().PatternType; left = left.cast(variables); right = right.cast(variables); - promote = Definition.BOOLEAN_TYPE; - actual = Definition.BOOLEAN_TYPE; + promote = variables.getDefinition().booleanType; + actual = variables.getDefinition().booleanType; } private void analyzeLSH(Locals variables) { left.analyze(variables); right.analyze(variables); - Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); - Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); + Type lhspromote = variables.getDefinition().caster.promoteNumeric(left.actual, false); + Type rhspromote = variables.getDefinition().caster.promoteNumeric(right.actual, false); if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply left shift [<<] to types " + @@ -383,7 +384,7 @@ public final class EBinary extends AExpression { left.expected = lhspromote; if (rhspromote.clazz == long.class) { - right.expected = Definition.INT_TYPE; + right.expected = variables.getDefinition().intType; right.explicit = true; } else { right.expected = rhspromote; @@ -410,8 +411,8 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); - Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); + Type lhspromote = variables.getDefinition().caster.promoteNumeric(left.actual, false); + Type rhspromote = variables.getDefinition().caster.promoteNumeric(right.actual, false); if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply right shift [>>] to types " + @@ -432,7 +433,7 @@ public final class EBinary extends AExpression { left.expected = lhspromote; if (rhspromote.clazz == long.class) { - right.expected = Definition.INT_TYPE; + right.expected = variables.getDefinition().intType; right.explicit = true; } else { right.expected = rhspromote; @@ -459,8 +460,8 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); - Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); + Type lhspromote = variables.getDefinition().caster.promoteNumeric(left.actual, false); + Type rhspromote = variables.getDefinition().caster.promoteNumeric(right.actual, false); actual = promote = lhspromote; shiftDistance = rhspromote; @@ -481,7 +482,7 @@ public final class EBinary extends AExpression { left.expected = lhspromote; if (rhspromote.clazz == long.class) { - right.expected = Definition.INT_TYPE; + right.expected = variables.getDefinition().intType; right.explicit = true; } else { right.expected = rhspromote; @@ -508,7 +509,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply and [&] to types " + @@ -549,7 +550,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteXor(left.actual, right.actual); + promote = variables.getDefinition().caster.promoteXor(left.actual, right.actual); if (promote == null) { throw createError(new ClassCastException("Cannot apply xor [^] to types " + @@ -591,7 +592,7 @@ public final class EBinary extends AExpression { left.analyze(variables); right.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); + promote = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply or [|] to types " + @@ -654,12 +655,12 @@ public final class EBinary extends AExpression { } else if (operation == Operation.FIND || operation == Operation.MATCH) { right.write(writer, globals); left.write(writer, globals); - writer.invokeVirtual(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_MATCHER); + writer.invokeVirtual(org.objectweb.asm.Type.getType(Pattern.class), WriterConstants.PATTERN_MATCHER); if (operation == Operation.FIND) { - writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_FIND); + writer.invokeVirtual(org.objectweb.asm.Type.getType(Matcher.class), WriterConstants.MATCHER_FIND); } else if (operation == Operation.MATCH) { - writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_MATCHES); + writer.invokeVirtual(org.objectweb.asm.Type.getType(Matcher.class), WriterConstants.MATCHER_MATCHES); } else { throw new IllegalStateException("Illegal tree structure."); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java index e1ca0b12292..dfc12423994 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java @@ -57,11 +57,11 @@ public final class EBool extends AExpression { @Override void analyze(Locals locals) { - left.expected = Definition.BOOLEAN_TYPE; + left.expected = locals.getDefinition().booleanType; left.analyze(locals); left = left.cast(locals); - right.expected = Definition.BOOLEAN_TYPE; + right.expected = locals.getDefinition().booleanType; right.analyze(locals); right = right.cast(locals); @@ -75,7 +75,7 @@ public final class EBool extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = locals.getDefinition().booleanType; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java index c9e9ba921d7..c180afa4df1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java @@ -49,7 +49,7 @@ public final class EBoolean extends AExpression { throw createError(new IllegalArgumentException("Must read from constant [" + constant + "].")); } - actual = Definition.BOOLEAN_TYPE; + actual = locals.getDefinition().booleanType; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java index 1229f8a45ce..564fcef8eef 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java @@ -34,7 +34,6 @@ import org.objectweb.asm.Type; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.Definition.VOID_TYPE; import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; /** @@ -83,11 +82,11 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) { Definition.Type from = ref.interfaceMethod.arguments.get(i); Definition.Type to = ref.delegateMethod.arguments.get(i); - AnalyzerCaster.getLegalCast(location, from, to, false, true); + locals.getDefinition().caster.getLegalCast(location, from, to, false, true); } - if (ref.interfaceMethod.rtn != VOID_TYPE) { - AnalyzerCaster.getLegalCast(location, ref.delegateMethod.rtn, ref.interfaceMethod.rtn, false, true); + if (ref.interfaceMethod.rtn.equals(locals.getDefinition().voidType) == false) { + locals.getDefinition().caster.getLegalCast(location, ref.delegateMethod.rtn, ref.interfaceMethod.rtn, false, true); } } catch (IllegalArgumentException e) { throw createError(e); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java index 11fe01da7c0..020ea48cd4c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java @@ -89,7 +89,7 @@ public final class EComp extends AExpression { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = variables.getDefinition().caster.promoteEquality(left.actual, right.actual); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply equals [==] to types " + @@ -133,14 +133,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeEqR(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = variables.getDefinition().caster.promoteEquality(left.actual, right.actual); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference equals [===] to types " + @@ -175,14 +175,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeNE(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = variables.getDefinition().caster.promoteEquality(left.actual, right.actual); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply not equals [!=] to types " + @@ -226,14 +226,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeNER(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = variables.getDefinition().caster.promoteEquality(left.actual, right.actual); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " + @@ -268,14 +268,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeGTE(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " + @@ -309,14 +309,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeGT(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than [>] to types " + @@ -350,14 +350,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeLTE(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " + @@ -391,14 +391,14 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } private void analyzeLT(Locals variables) { left.analyze(variables); right.analyze(variables); - promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = variables.getDefinition().caster.promoteNumeric(left.actual, right.actual, true); if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than [>=] to types " + @@ -432,7 +432,7 @@ public final class EComp extends AExpression { } } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java index 40f4b27ed15..571e57cad24 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java @@ -59,7 +59,7 @@ public final class EConditional extends AExpression { @Override void analyze(Locals locals) { - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); @@ -79,7 +79,7 @@ public final class EConditional extends AExpression { right.analyze(locals); if (expected == null) { - final Type promote = AnalyzerCaster.promoteConditional(left.actual, right.actual, left.constant, right.constant); + final Type promote = locals.getDefinition().caster.promoteConditional(left.actual, right.actual, left.constant, right.constant); left.expected = promote; right.expected = promote; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java index abe1fc48237..03038be3f7e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java @@ -48,23 +48,23 @@ final class EConstant extends AExpression { @Override void analyze(Locals locals) { if (constant instanceof String) { - actual = Definition.STRING_TYPE; + actual = locals.getDefinition().StringType; } else if (constant instanceof Double) { - actual = Definition.DOUBLE_TYPE; + actual = locals.getDefinition().doubleType; } else if (constant instanceof Float) { - actual = Definition.FLOAT_TYPE; + actual = locals.getDefinition().floatType; } else if (constant instanceof Long) { - actual = Definition.LONG_TYPE; + actual = locals.getDefinition().longType; } else if (constant instanceof Integer) { - actual = Definition.INT_TYPE; + actual = locals.getDefinition().intType; } else if (constant instanceof Character) { - actual = Definition.CHAR_TYPE; + actual = locals.getDefinition().charType; } else if (constant instanceof Short) { - actual = Definition.SHORT_TYPE; + actual = locals.getDefinition().shortType; } else if (constant instanceof Byte) { - actual = Definition.BYTE_TYPE; + actual = locals.getDefinition().byteType; } else if (constant instanceof Boolean) { - actual = Definition.BOOLEAN_TYPE; + actual = locals.getDefinition().booleanType; } else { throw createError(new IllegalStateException("Illegal tree structure.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java index 85675a02892..d6adbfb7ee0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java @@ -53,7 +53,7 @@ public final class EDecimal extends AExpression { if (value.endsWith("f") || value.endsWith("F")) { try { constant = Float.parseFloat(value.substring(0, value.length() - 1)); - actual = Definition.FLOAT_TYPE; + actual = locals.getDefinition().floatType; } catch (NumberFormatException exception) { throw createError(new IllegalArgumentException("Invalid float constant [" + value + "].")); } @@ -64,7 +64,7 @@ public final class EDecimal extends AExpression { } try { constant = Double.parseDouble(toParse); - actual = Definition.DOUBLE_TYPE; + actual = locals.getDefinition().doubleType; } catch (NumberFormatException exception) { throw createError(new IllegalArgumentException("Invalid double constant [" + value + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java index 7991759a916..e9816c524bf 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java @@ -81,7 +81,7 @@ public class EElvis extends AExpression { } if (expected == null) { - final Type promote = AnalyzerCaster.promoteConditional(lhs.actual, rhs.actual, lhs.constant, rhs.constant); + final Type promote = locals.getDefinition().caster.promoteConditional(lhs.actual, rhs.actual, lhs.constant, rhs.constant); lhs.expected = promote; rhs.expected = promote; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index c9b2a5d91d2..ffbb344f29c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -33,7 +33,6 @@ import org.objectweb.asm.Type; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.Definition.VOID_TYPE; import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; /** @@ -83,11 +82,11 @@ public final class EFunctionRef extends AExpression implements ILambda { for (int i = 0; i < interfaceMethod.arguments.size(); ++i) { Definition.Type from = interfaceMethod.arguments.get(i); Definition.Type to = delegateMethod.arguments.get(i); - AnalyzerCaster.getLegalCast(location, from, to, false, true); + locals.getDefinition().caster.getLegalCast(location, from, to, false, true); } - if (interfaceMethod.rtn != VOID_TYPE) { - AnalyzerCaster.getLegalCast(location, delegateMethod.rtn, interfaceMethod.rtn, false, true); + if (interfaceMethod.rtn.equals(locals.getDefinition().voidType) == false) { + locals.getDefinition().caster.getLegalCast(location, delegateMethod.rtn, interfaceMethod.rtn, false, true); } } else { // whitelist lookup diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java index c294c5177fe..a8c1217466b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java @@ -65,7 +65,7 @@ public final class EInstanceof extends AExpression { } // map to wrapped type for primitive types - resolvedType = type.clazz.isPrimitive() ? Definition.getBoxedType(type).clazz : type.clazz; + resolvedType = type.clazz.isPrimitive() ? locals.getDefinition().getBoxedType(type).clazz : type.clazz; // analyze and cast the expression expression.analyze(locals); @@ -75,9 +75,9 @@ public final class EInstanceof extends AExpression { // record if the expression returns a primitive primitiveExpression = expression.actual.clazz.isPrimitive(); // map to wrapped type for primitive types - expressionType = expression.actual.clazz.isPrimitive() ? Definition.getBoxedType(expression.actual).clazz : type.clazz; + expressionType = expression.actual.clazz.isPrimitive() ? locals.getDefinition().getBoxedType(expression.actual).clazz : type.clazz; - actual = Definition.BOOLEAN_TYPE; + actual = locals.getDefinition().booleanType; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java index 748ff67ddd4..07de9138e7c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -39,10 +39,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.Definition.VOID_TYPE; -import static org.elasticsearch.painless.WriterConstants.CLASS_NAME; import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; -import static org.objectweb.asm.Opcodes.H_INVOKESTATIC; /** * Lambda expression node. @@ -111,7 +108,7 @@ public final class ELambda extends AExpression implements ILambda { if (expected == null) { interfaceMethod = null; // we don't know anything: treat as def - returnType = Definition.DEF_TYPE; + returnType = locals.getDefinition().DefType; // don't infer any types, replace any null types with def actualParamTypeStrs = new ArrayList<>(paramTypeStrs.size()); for (String type : paramTypeStrs) { @@ -133,8 +130,8 @@ public final class ELambda extends AExpression implements ILambda { throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name + "] in [" + expected.clazz + "]"); // for method invocation, its allowed to ignore the return value - if (interfaceMethod.rtn == Definition.VOID_TYPE) { - returnType = Definition.DEF_TYPE; + if (interfaceMethod.rtn.equals(locals.getDefinition().voidType)) { + returnType = locals.getDefinition().DefType; } else { returnType = interfaceMethod.rtn; } @@ -195,11 +192,11 @@ public final class ELambda extends AExpression implements ILambda { for (int i = 0; i < interfaceMethod.arguments.size(); ++i) { Type from = interfaceMethod.arguments.get(i); Type to = desugared.parameters.get(i + captures.size()).type; - AnalyzerCaster.getLegalCast(location, from, to, false, true); + locals.getDefinition().caster.getLegalCast(location, from, to, false, true); } - if (interfaceMethod.rtn != VOID_TYPE) { - AnalyzerCaster.getLegalCast(location, desugared.rtnType, interfaceMethod.rtn, false, true); + if (interfaceMethod.rtn.equals(locals.getDefinition().voidType) == false) { + locals.getDefinition().caster.getLegalCast(location, desugared.rtnType, interfaceMethod.rtn, false, true); } actual = expected; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java index 999d35551ce..44b4a67b395 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java @@ -58,7 +58,7 @@ public final class EListInit extends AExpression { throw createError(new IllegalArgumentException("Must read from list initializer.")); } - actual = Definition.ARRAY_LIST_TYPE; + actual = locals.getDefinition().ArrayListType; constructor = actual.struct.constructors.get(new MethodKey("", 0)); @@ -75,7 +75,7 @@ public final class EListInit extends AExpression { for (int index = 0; index < values.size(); ++index) { AExpression expression = values.get(index); - expression.expected = Definition.DEF_TYPE; + expression.expected = locals.getDefinition().DefType; expression.internal = true; expression.analyze(locals); values.set(index, expression.cast(locals)); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java index 0647b5716e0..42fb37a2407 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java @@ -64,7 +64,7 @@ public final class EMapInit extends AExpression { throw createError(new IllegalArgumentException("Must read from map initializer.")); } - actual = Definition.HASH_MAP_TYPE; + actual = locals.getDefinition().HashMapType; constructor = actual.struct.constructors.get(new MethodKey("", 0)); @@ -85,7 +85,7 @@ public final class EMapInit extends AExpression { for (int index = 0; index < keys.size(); ++index) { AExpression expression = keys.get(index); - expression.expected = Definition.DEF_TYPE; + expression.expected = locals.getDefinition().DefType; expression.internal = true; expression.analyze(locals); keys.set(index, expression.cast(locals)); @@ -94,7 +94,7 @@ public final class EMapInit extends AExpression { for (int index = 0; index < values.size(); ++index) { AExpression expression = values.get(index); - expression.expected = Definition.DEF_TYPE; + expression.expected = locals.getDefinition().DefType; expression.internal = true; expression.analyze(locals); values.set(index, expression.cast(locals)); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java index d32a153b797..51198e03bde 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java @@ -71,8 +71,7 @@ public final class ENewArray extends AExpression { for (int argument = 0; argument < arguments.size(); ++argument) { AExpression expression = arguments.get(argument); - expression.expected = initialize ? locals.getDefinition().getType(type.struct, 0) - : Definition.INT_TYPE; + expression.expected = initialize ? locals.getDefinition().getType(type.struct, 0) : locals.getDefinition().intType; expression.internal = true; expression.analyze(locals); arguments.set(argument, expression.cast(locals)); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java index 6d6672b183e..73d34e7eae9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java @@ -58,7 +58,7 @@ public final class ENull extends AExpression { actual = expected; } else { - actual = Definition.OBJECT_TYPE; + actual = locals.getDefinition().ObjectType; } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java index d047aaa6355..02df9c10a13 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java @@ -62,7 +62,7 @@ public final class ENumeric extends AExpression { try { constant = Double.parseDouble(value.substring(0, value.length() - 1)); - actual = Definition.DOUBLE_TYPE; + actual = locals.getDefinition().doubleType; } catch (NumberFormatException exception) { throw createError(new IllegalArgumentException("Invalid double constant [" + value + "].")); } @@ -73,14 +73,14 @@ public final class ENumeric extends AExpression { try { constant = Float.parseFloat(value.substring(0, value.length() - 1)); - actual = Definition.FLOAT_TYPE; + actual = locals.getDefinition().floatType; } catch (NumberFormatException exception) { throw createError(new IllegalArgumentException("Invalid float constant [" + value + "].")); } } else if (value.endsWith("l") || value.endsWith("L")) { try { constant = Long.parseLong(value.substring(0, value.length() - 1), radix); - actual = Definition.LONG_TYPE; + actual = locals.getDefinition().longType; } catch (NumberFormatException exception) { throw createError(new IllegalArgumentException("Invalid long constant [" + value + "].")); } @@ -91,16 +91,16 @@ public final class ENumeric extends AExpression { if (sort == byte.class && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) { constant = (byte)integer; - actual = Definition.BYTE_TYPE; + actual = locals.getDefinition().byteType; } else if (sort == char.class && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) { constant = (char)integer; - actual = Definition.CHAR_TYPE; + actual = locals.getDefinition().charType; } else if (sort == short.class && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) { constant = (short)integer; - actual = Definition.SHORT_TYPE; + actual = locals.getDefinition().shortType; } else { constant = integer; - actual = Definition.INT_TYPE; + actual = locals.getDefinition().intType; } } catch (NumberFormatException exception) { try { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java index 4b38868b1b1..4fff59f9d59 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java @@ -73,22 +73,23 @@ public final class ERegex extends AExpression { new IllegalArgumentException("Error compiling regex: " + e.getDescription())); } - constant = new Constant(location, Definition.PATTERN_TYPE.type, "regexAt$" + location.getOffset(), this::initializeConstant); - actual = Definition.PATTERN_TYPE; + constant = new Constant( + location, locals.getDefinition().PatternType.type, "regexAt$" + location.getOffset(), this::initializeConstant); + actual = locals.getDefinition().PatternType; } @Override void write(MethodWriter writer, Globals globals) { writer.writeDebugInfo(location); - writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, Definition.PATTERN_TYPE.type); + writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, org.objectweb.asm.Type.getType(Pattern.class)); globals.addConstantInitializer(constant); } private void initializeConstant(MethodWriter writer) { writer.push(pattern); writer.push(flags); - writer.invokeStatic(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE); + writer.invokeStatic(org.objectweb.asm.Type.getType(Pattern.class), WriterConstants.PATTERN_COMPILE); } private int flagForChar(char c) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java index 0c2cec13e24..001e9309628 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java @@ -50,7 +50,7 @@ public final class EString extends AExpression { throw createError(new IllegalArgumentException("Must read from constant [" + constant + "].")); } - actual = Definition.STRING_TYPE; + actual = locals.getDefinition().StringType; } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java index 23b59bac83b..e9971b538f5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java @@ -76,7 +76,7 @@ public final class EUnary extends AExpression { } void analyzeNot(Locals variables) { - child.expected = Definition.BOOLEAN_TYPE; + child.expected = variables.getDefinition().booleanType; child.analyze(variables); child = child.cast(variables); @@ -84,13 +84,13 @@ public final class EUnary extends AExpression { constant = !(boolean)child.constant; } - actual = Definition.BOOLEAN_TYPE; + actual = variables.getDefinition().booleanType; } void analyzeBWNot(Locals variables) { child.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(child.actual, false); + promote = variables.getDefinition().caster.promoteNumeric(child.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply not [~] to type [" + child.actual.name + "].")); @@ -121,7 +121,7 @@ public final class EUnary extends AExpression { void analyzerAdd(Locals variables) { child.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(child.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(child.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply positive [+] to type [" + child.actual.name + "].")); @@ -156,7 +156,7 @@ public final class EUnary extends AExpression { void analyzerSub(Locals variables) { child.analyze(variables); - promote = AnalyzerCaster.promoteNumeric(child.actual, true); + promote = variables.getDefinition().caster.promoteNumeric(child.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply negative [-] to type [" + child.actual.name + "].")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java index 56366262900..d9e74a70d94 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java @@ -73,7 +73,7 @@ public final class PCallInvoke extends AExpression { Struct struct = prefix.actual.struct; if (prefix.actual.clazz.isPrimitive()) { - struct = Definition.getBoxedType(prefix.actual).struct; + struct = locals.getDefinition().getBoxedType(prefix.actual).struct; } MethodKey methodKey = new MethodKey(name, arguments.size()); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java index 806f971850f..a308bc2c6c9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java @@ -56,7 +56,7 @@ final class PSubArrayLength extends AStoreable { throw createError(new IllegalArgumentException("Cannot write to read-only field [length] for an array.")); } - actual = Definition.INT_TYPE; + actual = locals.getDefinition().intType; } else { throw createError(new IllegalArgumentException("Field [" + value + "] does not exist for type [" + type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java index 1f3d4109bca..56a29f7930b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java @@ -51,7 +51,7 @@ final class PSubBrace extends AStoreable { @Override void analyze(Locals locals) { - index.expected = Definition.INT_TYPE; + index.expected = locals.getDefinition().intType; index.analyze(locals); index = index.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java index c13c1bcb992..cac80f80516 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java @@ -53,7 +53,7 @@ final class PSubDefArray extends AStoreable { index.expected = index.actual; index = index.cast(locals); - actual = expected == null || explicit ? Definition.DEF_TYPE : expected; + actual = expected == null || explicit ? locals.getDefinition().DefType : expected; } @Override @@ -83,7 +83,7 @@ final class PSubDefArray extends AStoreable { writer.dup(); // def, def index.write(writer, globals); // def, def, unnormalized_index org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType( - index.actual.type, Definition.DEF_TYPE.type, index.actual.type); + index.actual.type, org.objectweb.asm.Type.getType(Object.class), index.actual.type); writer.invokeDefCall("normalizeIndex", methodType, DefBootstrap.INDEX_NORMALIZE); // def, normalized_index } @@ -92,7 +92,7 @@ final class PSubDefArray extends AStoreable { writer.writeDebugInfo(location); org.objectweb.asm.Type methodType = - org.objectweb.asm.Type.getMethodType(actual.type, Definition.DEF_TYPE.type, index.actual.type); + org.objectweb.asm.Type.getMethodType(actual.type, org.objectweb.asm.Type.getType(Object.class), index.actual.type); writer.invokeDefCall("arrayLoad", methodType, DefBootstrap.ARRAY_LOAD); } @@ -101,7 +101,8 @@ final class PSubDefArray extends AStoreable { writer.writeDebugInfo(location); org.objectweb.asm.Type methodType = - org.objectweb.asm.Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, actual.type); + org.objectweb.asm.Type.getMethodType( + org.objectweb.asm.Type.getType(void.class), org.objectweb.asm.Type.getType(Object.class), index.actual.type, actual.type); writer.invokeDefCall("arrayStore", methodType, DefBootstrap.ARRAY_STORE); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java index 2d0734915af..89fc169704f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java @@ -80,7 +80,7 @@ final class PSubDefCall extends AExpression { arguments.set(argument, expression.cast(locals)); } - actual = expected == null || explicit ? Definition.DEF_TYPE : expected; + actual = expected == null || explicit ? locals.getDefinition().DefType : expected; } @Override @@ -90,7 +90,7 @@ final class PSubDefCall extends AExpression { List parameterTypes = new ArrayList<>(); // first parameter is the receiver, we never know its type: always Object - parameterTypes.add(Definition.DEF_TYPE.type); + parameterTypes.add(org.objectweb.asm.Type.getType(Object.class)); // append each argument for (AExpression argument : arguments) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java index 24c6c91422d..c1f0c468e42 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java @@ -50,7 +50,7 @@ final class PSubDefField extends AStoreable { @Override void analyze(Locals locals) { - actual = expected == null || explicit ? Definition.DEF_TYPE : expected; + actual = expected == null || explicit ? locals.getDefinition().DefType : expected; } @Override @@ -58,7 +58,7 @@ final class PSubDefField extends AStoreable { writer.writeDebugInfo(location); org.objectweb.asm.Type methodType = - org.objectweb.asm.Type.getMethodType(actual.type, Definition.DEF_TYPE.type); + org.objectweb.asm.Type.getMethodType(actual.type, org.objectweb.asm.Type.getType(Object.class)); writer.invokeDefCall(value, methodType, DefBootstrap.LOAD); } @@ -87,7 +87,7 @@ final class PSubDefField extends AStoreable { writer.writeDebugInfo(location); org.objectweb.asm.Type methodType = - org.objectweb.asm.Type.getMethodType(actual.type, Definition.DEF_TYPE.type); + org.objectweb.asm.Type.getMethodType(actual.type, org.objectweb.asm.Type.getType(Object.class)); writer.invokeDefCall(value, methodType, DefBootstrap.LOAD); } @@ -95,8 +95,8 @@ final class PSubDefField extends AStoreable { void store(MethodWriter writer, Globals globals) { writer.writeDebugInfo(location); - org.objectweb.asm.Type methodType = - org.objectweb.asm.Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, actual.type); + org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType( + org.objectweb.asm.Type.getType(void.class), org.objectweb.asm.Type.getType(Object.class), actual.type); writer.invokeDefCall(value, methodType, DefBootstrap.STORE); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java index 343fbf18276..557c1aed693 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java @@ -75,7 +75,7 @@ final class PSubListShortcut extends AStoreable { } if ((read || write) && (!read || getter != null) && (!write || setter != null)) { - index.expected = Definition.INT_TYPE; + index.expected = locals.getDefinition().intType; index.analyze(locals); index = index.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java index 979bd4eec9f..858dc43b6c8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java @@ -73,7 +73,7 @@ public final class SDo extends AStatement { throw createError(new IllegalArgumentException("Extraneous do while loop.")); } - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java index a511cb82429..378ecb6b4f8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java @@ -94,7 +94,7 @@ public final class SFor extends AStatement { } if (condition != null) { - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java index 08495cd8d7f..e5d2233fa98 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java @@ -56,7 +56,7 @@ public final class SIf extends AStatement { @Override void analyze(Locals locals) { - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java index 4feb6b3fe62..d5dd8346485 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java @@ -65,7 +65,7 @@ public final class SIfElse extends AStatement { @Override void analyze(Locals locals) { - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java index 2083d3ddbe5..09c73c525be 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java @@ -66,11 +66,11 @@ final class SSubEachArray extends AStatement { // also add the location offset to make the names unique in case of nested for each loops. array = locals.addVariable(location, expression.actual, "#array" + location.getOffset(), true); - index = locals.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), + index = locals.addVariable(location, locals.getDefinition().intType, "#index" + location.getOffset(), true); indexed = locals.getDefinition().getType(expression.actual.struct, expression.actual.dimensions - 1); - cast = AnalyzerCaster.getLegalCast(location, indexed, variable.type, true, true); + cast = locals.getDefinition().caster.getLegalCast(location, indexed, variable.type, true, true); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java index 566003fd171..a51a459f0f3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java @@ -33,6 +33,7 @@ import org.elasticsearch.painless.MethodWriter; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; +import java.util.Iterator; import java.util.Objects; import java.util.Set; @@ -84,7 +85,7 @@ final class SSubEachIterable extends AStatement { } } - cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, variable.type, true, true); + cast = locals.getDefinition().caster.getLegalCast(location, locals.getDefinition().DefType, variable.type, true, true); } @Override @@ -95,7 +96,7 @@ final class SSubEachIterable extends AStatement { if (method == null) { org.objectweb.asm.Type methodType = org.objectweb.asm.Type - .getMethodType(Definition.ITERATOR_TYPE.type, Definition.DEF_TYPE.type); + .getMethodType(org.objectweb.asm.Type.getType(Iterator.class), org.objectweb.asm.Type.getType(Object.class)); writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR); } else { method.write(writer); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java index e87d862b2ac..1e96586b6f7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java @@ -48,7 +48,7 @@ public final class SThrow extends AStatement { @Override void analyze(Locals locals) { - expression.expected = Definition.EXCEPTION_TYPE; + expression.expected = locals.getDefinition().ExceptionType; expression.analyze(locals); expression = expression.cast(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java index 7663c853a0c..f6c750a94a1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java @@ -59,7 +59,7 @@ public final class SWhile extends AStatement { void analyze(Locals locals) { locals = Locals.newLocalScope(locals); - condition.expected = Definition.BOOLEAN_TYPE; + condition.expected = locals.getDefinition().booleanType; condition.analyze(locals); condition = condition.cast(locals); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java index e8d2f1332e0..249ce122b43 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java @@ -23,13 +23,6 @@ import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.painless.Definition.BYTE_TYPE; -import static org.elasticsearch.painless.Definition.DOUBLE_TYPE; -import static org.elasticsearch.painless.Definition.FLOAT_TYPE; -import static org.elasticsearch.painless.Definition.INT_TYPE; -import static org.elasticsearch.painless.Definition.LONG_TYPE; -import static org.elasticsearch.painless.Definition.SHORT_TYPE; - public class AnalyzerCasterTests extends ESTestCase { private static void assertCast(Type actual, Type expected, boolean mustBeExplicit) { @@ -37,68 +30,68 @@ public class AnalyzerCasterTests extends ESTestCase { if (actual.equals(expected)) { assertFalse(mustBeExplicit); - assertNull(AnalyzerCaster.getLegalCast(location, actual, expected, false, false)); - assertNull(AnalyzerCaster.getLegalCast(location, actual, expected, true, false)); + assertNull(Definition.DEFINITION.caster.getLegalCast(location, actual, expected, false, false)); + assertNull(Definition.DEFINITION.caster.getLegalCast(location, actual, expected, true, false)); return; } - Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, true, false); + Cast cast = Definition.DEFINITION.caster.getLegalCast(location, actual, expected, true, false); assertEquals(actual, cast.from); assertEquals(expected, cast.to); if (mustBeExplicit) { ClassCastException error = expectThrows(ClassCastException.class, - () -> AnalyzerCaster.getLegalCast(location, actual, expected, false, false)); + () -> Definition.DEFINITION.caster.getLegalCast(location, actual, expected, false, false)); assertTrue(error.getMessage().startsWith("Cannot cast")); } else { - cast = AnalyzerCaster.getLegalCast(location, actual, expected, false, false); + cast = Definition.DEFINITION.caster.getLegalCast(location, actual, expected, false, false); assertEquals(actual, cast.from); assertEquals(expected, cast.to); } } public void testNumericCasts() { - assertCast(BYTE_TYPE, BYTE_TYPE, false); - assertCast(BYTE_TYPE, SHORT_TYPE, false); - assertCast(BYTE_TYPE, INT_TYPE, false); - assertCast(BYTE_TYPE, LONG_TYPE, false); - assertCast(BYTE_TYPE, FLOAT_TYPE, false); - assertCast(BYTE_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.byteType, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.shortType, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.intType, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.longType, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.floatType, false); + assertCast(Definition.DEFINITION.byteType, Definition.DEFINITION.doubleType, false); - assertCast(SHORT_TYPE, BYTE_TYPE, true); - assertCast(SHORT_TYPE, SHORT_TYPE, false); - assertCast(SHORT_TYPE, INT_TYPE, false); - assertCast(SHORT_TYPE, LONG_TYPE, false); - assertCast(SHORT_TYPE, FLOAT_TYPE, false); - assertCast(SHORT_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.byteType, true); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.shortType, false); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.intType, false); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.longType, false); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.floatType, false); + assertCast(Definition.DEFINITION.shortType, Definition.DEFINITION.doubleType, false); - assertCast(INT_TYPE, BYTE_TYPE, true); - assertCast(INT_TYPE, SHORT_TYPE, true); - assertCast(INT_TYPE, INT_TYPE, false); - assertCast(INT_TYPE, LONG_TYPE, false); - assertCast(INT_TYPE, FLOAT_TYPE, false); - assertCast(INT_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.byteType, true); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.shortType, true); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.intType, false); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.longType, false); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.floatType, false); + assertCast(Definition.DEFINITION.intType, Definition.DEFINITION.doubleType, false); - assertCast(LONG_TYPE, BYTE_TYPE, true); - assertCast(LONG_TYPE, SHORT_TYPE, true); - assertCast(LONG_TYPE, INT_TYPE, true); - assertCast(LONG_TYPE, LONG_TYPE, false); - assertCast(LONG_TYPE, FLOAT_TYPE, false); - assertCast(LONG_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.byteType, true); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.shortType, true); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.intType, true); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.longType, false); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.floatType, false); + assertCast(Definition.DEFINITION.longType, Definition.DEFINITION.doubleType, false); - assertCast(FLOAT_TYPE, BYTE_TYPE, true); - assertCast(FLOAT_TYPE, SHORT_TYPE, true); - assertCast(FLOAT_TYPE, INT_TYPE, true); - assertCast(FLOAT_TYPE, LONG_TYPE, true); - assertCast(FLOAT_TYPE, FLOAT_TYPE, false); - assertCast(FLOAT_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.byteType, true); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.shortType, true); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.intType, true); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.longType, true); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.floatType, false); + assertCast(Definition.DEFINITION.floatType, Definition.DEFINITION.doubleType, false); - assertCast(DOUBLE_TYPE, BYTE_TYPE, true); - assertCast(DOUBLE_TYPE, SHORT_TYPE, true); - assertCast(DOUBLE_TYPE, INT_TYPE, true); - assertCast(DOUBLE_TYPE, LONG_TYPE, true); - assertCast(DOUBLE_TYPE, FLOAT_TYPE, true); - assertCast(DOUBLE_TYPE, DOUBLE_TYPE, false); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.byteType, true); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.shortType, true); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.intType, true); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.longType, true); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.floatType, true); + assertCast(Definition.DEFINITION.doubleType, Definition.DEFINITION.doubleType, false); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java index e4f04b23529..cdd51447bc1 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java @@ -66,7 +66,7 @@ public class BaseClassTests extends ScriptTestCase { } public void testGets() { - Compiler compiler = new Compiler(Gets.class, Definition.BUILTINS); + Compiler compiler = new Compiler(Gets.class, Definition.DEFINITION); Map map = new HashMap<>(); map.put("s", 1); @@ -84,7 +84,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(); } public void testNoArgs() { - Compiler compiler = new Compiler(NoArgs.class, Definition.BUILTINS); + Compiler compiler = new Compiler(NoArgs.class, Definition.DEFINITION); assertEquals(1, ((NoArgs)scriptEngine.compile(compiler, null, "1", emptyMap())).execute()); assertEquals("foo", ((NoArgs)scriptEngine.compile(compiler, null, "'foo'", emptyMap())).execute()); @@ -108,13 +108,13 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(Object arg); } public void testOneArg() { - Compiler compiler = new Compiler(OneArg.class, Definition.BUILTINS); + Compiler compiler = new Compiler(OneArg.class, Definition.DEFINITION); Object rando = randomInt(); assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando)); rando = randomAlphaOfLength(5); assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando)); - Compiler noargs = new Compiler(NoArgs.class, Definition.BUILTINS); + Compiler noargs = new Compiler(NoArgs.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, () -> scriptEngine.compile(noargs, null, "doc", emptyMap())); assertEquals("Variable [doc] is not defined.", e.getMessage()); @@ -129,7 +129,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(String[] arg); } public void testArrayArg() { - Compiler compiler = new Compiler(ArrayArg.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ArrayArg.class, Definition.DEFINITION); String rando = randomAlphaOfLength(5); assertEquals(rando, ((ArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new String[] {rando, "foo"})); } @@ -139,7 +139,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(int[] arg); } public void testPrimitiveArrayArg() { - Compiler compiler = new Compiler(PrimitiveArrayArg.class, Definition.BUILTINS); + Compiler compiler = new Compiler(PrimitiveArrayArg.class, Definition.DEFINITION); int rando = randomInt(); assertEquals(rando, ((PrimitiveArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new int[] {rando, 10})); } @@ -149,7 +149,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(Object[] arg); } public void testDefArrayArg() { - Compiler compiler = new Compiler(DefArrayArg.class, Definition.BUILTINS); + Compiler compiler = new Compiler(DefArrayArg.class, Definition.DEFINITION); Object rando = randomInt(); assertEquals(rando, ((DefArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new Object[] {rando, 10})); rando = randomAlphaOfLength(5); @@ -167,7 +167,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract boolean needsD(); } public void testManyArgs() { - Compiler compiler = new Compiler(ManyArgs.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ManyArgs.class, Definition.DEFINITION); int rando = randomInt(); assertEquals(rando, ((ManyArgs)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0)); assertEquals(10, ((ManyArgs)scriptEngine.compile(compiler, null, "a + b + c + d", emptyMap())).execute(1, 2, 3, 4)); @@ -195,7 +195,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(String... arg); } public void testVararg() { - Compiler compiler = new Compiler(VarargTest.class, Definition.BUILTINS); + Compiler compiler = new Compiler(VarargTest.class, Definition.DEFINITION); assertEquals("foo bar baz", ((VarargTest)scriptEngine.compile(compiler, null, "String.join(' ', Arrays.asList(arg))", emptyMap())) .execute("foo", "bar", "baz")); } @@ -211,7 +211,7 @@ public class BaseClassTests extends ScriptTestCase { } } public void testDefaultMethods() { - Compiler compiler = new Compiler(DefaultMethods.class, Definition.BUILTINS); + Compiler compiler = new Compiler(DefaultMethods.class, Definition.DEFINITION); int rando = randomInt(); assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0)); assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).executeWithASingleOne(rando, 0, 0)); @@ -225,7 +225,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract void execute(Map map); } public void testReturnsVoid() { - Compiler compiler = new Compiler(ReturnsVoid.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ReturnsVoid.class, Definition.DEFINITION); Map map = new HashMap<>(); ((ReturnsVoid)scriptEngine.compile(compiler, null, "map.a = 'foo'", emptyMap())).execute(map); assertEquals(singletonMap("a", "foo"), map); @@ -244,7 +244,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract boolean execute(); } public void testReturnsPrimitiveBoolean() { - Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, Definition.DEFINITION); assertEquals(true, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "true", emptyMap())).execute()); assertEquals(false, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "false", emptyMap())).execute()); @@ -286,7 +286,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract int execute(); } public void testReturnsPrimitiveInt() { - Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, Definition.DEFINITION); assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "1", emptyMap())).execute()); assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "(int) 1L", emptyMap())).execute()); @@ -328,7 +328,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract float execute(); } public void testReturnsPrimitiveFloat() { - Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, Definition.DEFINITION); assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "1.1f", emptyMap())).execute(), 0); assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "(float) 1.1d", emptyMap())).execute(), 0); @@ -359,7 +359,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract double execute(); } public void testReturnsPrimitiveDouble() { - Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, Definition.BUILTINS); + Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, Definition.DEFINITION); assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1", emptyMap())).execute(), 0); assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1L", emptyMap())).execute(), 0); @@ -393,7 +393,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(String foo); } public void testNoArgumentsConstant() { - Compiler compiler = new Compiler(NoArgumentsConstant.class, Definition.BUILTINS); + Compiler compiler = new Compiler(NoArgumentsConstant.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith( @@ -406,7 +406,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(String foo); } public void testWrongArgumentsConstant() { - Compiler compiler = new Compiler(WrongArgumentsConstant.class, Definition.BUILTINS); + Compiler compiler = new Compiler(WrongArgumentsConstant.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith( @@ -419,7 +419,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(String foo); } public void testWrongLengthOfArgumentConstant() { - Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, Definition.BUILTINS); + Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith("[" + WrongLengthOfArgumentConstant.class.getName() + "#ARGUMENTS] has length [2] but [" @@ -431,7 +431,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(UnknownArgType foo); } public void testUnknownArgType() { - Compiler compiler = new Compiler(UnknownArgType.class, Definition.BUILTINS); + Compiler compiler = new Compiler(UnknownArgType.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("[foo] is of unknown type [" + UnknownArgType.class.getName() + ". Painless interfaces can only accept arguments " @@ -443,7 +443,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract UnknownReturnType execute(String foo); } public void testUnknownReturnType() { - Compiler compiler = new Compiler(UnknownReturnType.class, Definition.BUILTINS); + Compiler compiler = new Compiler(UnknownReturnType.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("Painless can only implement execute methods returning a whitelisted type but [" + UnknownReturnType.class.getName() @@ -455,7 +455,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(UnknownArgTypeInArray[] foo); } public void testUnknownArgTypeInArray() { - Compiler compiler = new Compiler(UnknownArgTypeInArray.class, Definition.BUILTINS); + Compiler compiler = new Compiler(UnknownArgTypeInArray.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("[foo] is of unknown type [" + UnknownArgTypeInArray.class.getName() + ". Painless interfaces can only accept " @@ -467,7 +467,7 @@ public class BaseClassTests extends ScriptTestCase { public abstract Object execute(boolean foo); } public void testTwoExecuteMethods() { - Compiler compiler = new Compiler(TwoExecuteMethods.class, Definition.BUILTINS); + Compiler compiler = new Compiler(TwoExecuteMethods.class, Definition.DEFINITION); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "null", emptyMap())); assertEquals("Painless can only implement interfaces that have a single method named [execute] but [" diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java index c1098a1e7af..fe50578df1a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java @@ -34,7 +34,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; public class DebugTests extends ScriptTestCase { - private final Definition definition = Definition.BUILTINS; + private final Definition definition = Definition.DEFINITION; public void testExplain() { // Debug.explain can explain an object diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java index 1169525b9b8..2b4a896fb5e 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java @@ -39,7 +39,7 @@ final class Debugger { PrintWriter outputWriter = new PrintWriter(output); Textifier textifier = new Textifier(); try { - new Compiler(iface, Definition.BUILTINS).compile("", source, settings, textifier); + new Compiler(iface, Definition.DEFINITION).compile("", source, settings, textifier); } catch (Exception e) { textifier.print(outputWriter); e.addSuppressed(new Exception("current bytecode: \n" + output)); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java index 24e096d03a0..7188caf4251 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java @@ -30,14 +30,14 @@ import java.util.HashMap; import org.elasticsearch.test.ESTestCase; public class DefBootstrapTests extends ESTestCase { - private final Definition definition = Definition.BUILTINS; - + private final Definition definition = Definition.DEFINITION; + /** calls toString() on integers, twice */ public void testOneType() throws Throwable { CallSite site = DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), - "toString", - MethodType.methodType(String.class, Object.class), + MethodHandles.publicLookup(), + "toString", + MethodType.methodType(String.class, Object.class), 0, DefBootstrap.METHOD_CALL, ""); @@ -52,12 +52,12 @@ public class DefBootstrapTests extends ESTestCase { assertEquals("6", (String)handle.invokeExact((Object)6)); assertDepthEquals(site, 1); } - + public void testTwoTypes() throws Throwable { CallSite site = DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), - "toString", - MethodType.methodType(String.class, Object.class), + MethodHandles.publicLookup(), + "toString", + MethodType.methodType(String.class, Object.class), 0, DefBootstrap.METHOD_CALL, ""); @@ -75,14 +75,14 @@ public class DefBootstrapTests extends ESTestCase { assertEquals("2.5", (String)handle.invokeExact((Object)2.5f)); assertDepthEquals(site, 2); } - + public void testTooManyTypes() throws Throwable { // if this changes, test must be rewritten assertEquals(5, DefBootstrap.PIC.MAX_DEPTH); CallSite site = DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), - "toString", - MethodType.methodType(String.class, Object.class), + MethodHandles.publicLookup(), + "toString", + MethodType.methodType(String.class, Object.class), 0, DefBootstrap.METHOD_CALL, ""); @@ -102,13 +102,13 @@ public class DefBootstrapTests extends ESTestCase { assertEquals("c", (String)handle.invokeExact((Object)'c')); assertDepthEquals(site, 5); } - + /** test that we revert to the megamorphic classvalue cache and that it works as expected */ public void testMegamorphic() throws Throwable { - DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), - "size", - MethodType.methodType(int.class, Object.class), + DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(definition, + MethodHandles.publicLookup(), + "size", + MethodType.methodType(int.class, Object.class), 0, DefBootstrap.METHOD_CALL, ""); @@ -118,12 +118,12 @@ public class DefBootstrapTests extends ESTestCase { assertEquals(1, (int)handle.invokeExact((Object) Collections.singletonMap("a", "b"))); assertEquals(3, (int)handle.invokeExact((Object) Arrays.asList("x", "y", "z"))); assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("u", "v"))); - + final HashMap map = new HashMap<>(); map.put("x", "y"); map.put("a", "b"); assertEquals(2, (int)handle.invokeExact((Object) map)); - + final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> { Integer.toString((int)handle.invokeExact(new Object())); }); @@ -133,12 +133,12 @@ public class DefBootstrapTests extends ESTestCase { e.getClassName().startsWith("org.elasticsearch.painless.DefBootstrap$PIC$"); })); } - + // test operators with null guards public void testNullGuardAdd() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), + MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, Object.class, Object.class), 0, @@ -147,7 +147,7 @@ public class DefBootstrapTests extends ESTestCase { MethodHandle handle = site.dynamicInvoker(); assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test")); } - + public void testNullGuardAddWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, MethodHandles.publicLookup(), @@ -160,11 +160,11 @@ public class DefBootstrapTests extends ESTestCase { assertEquals(2, (Object)handle.invokeExact((Object)1, (Object)1)); assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test")); } - + public void testNullGuardEq() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), - "eq", + MethodHandles.publicLookup(), + "eq", MethodType.methodType(boolean.class, Object.class, Object.class), 0, DefBootstrap.BINARY_OPERATOR, @@ -173,7 +173,7 @@ public class DefBootstrapTests extends ESTestCase { assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test")); assertTrue((boolean) handle.invokeExact((Object)null, (Object)null)); } - + public void testNullGuardEqWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, MethodHandles.publicLookup(), @@ -187,14 +187,14 @@ public class DefBootstrapTests extends ESTestCase { assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test")); assertTrue((boolean) handle.invokeExact((Object)null, (Object)null)); } - + // make sure these operators work without null guards too // for example, nulls are only legal for + if the other parameter is a String, // and can be disabled in some circumstances. - + public void testNoNullGuardAdd() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, - MethodHandles.publicLookup(), + MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, int.class, Object.class), 0, @@ -205,7 +205,7 @@ public class DefBootstrapTests extends ESTestCase { assertNotNull((Object)handle.invokeExact(5, (Object)null)); }); } - + public void testNoNullGuardAddWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, MethodHandles.publicLookup(), @@ -220,7 +220,7 @@ public class DefBootstrapTests extends ESTestCase { assertNotNull((Object)handle.invokeExact(5, (Object)null)); }); } - + static void assertDepthEquals(CallSite site, int expected) { DefBootstrap.PIC dsite = (DefBootstrap.PIC) site; assertEquals(expected, dsite.depth); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java index 929d9486d88..72d5af1942a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java @@ -52,7 +52,7 @@ public class OrTests extends ScriptTestCase { assertEquals(5L | -12L, exec("return 5L | -12L;")); assertEquals(7L | 15L | 3L, exec("return 7L | 15L | 3L;")); } - + public void testIllegal() throws Exception { expectScriptThrows(ClassCastException.class, () -> { exec("float x = (float)4; int y = 1; return x | y"); @@ -61,7 +61,7 @@ public class OrTests extends ScriptTestCase { exec("double x = (double)4; int y = 1; return x | y"); }); } - + public void testDef() { expectScriptThrows(ClassCastException.class, () -> { exec("def x = (float)4; def y = (byte)1; return x | y"); @@ -104,13 +104,13 @@ public class OrTests extends ScriptTestCase { assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y")); assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y")); assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y")); - + assertEquals(true, exec("def x = true; def y = true; return x | y")); assertEquals(true, exec("def x = true; def y = false; return x | y")); assertEquals(true, exec("def x = false; def y = true; return x | y")); assertEquals(false, exec("def x = false; def y = false; return x | y")); } - + public void testDefTypedLHS() { expectScriptThrows(ClassCastException.class, () -> { exec("float x = (float)4; def y = (byte)1; return x | y"); @@ -153,13 +153,13 @@ public class OrTests extends ScriptTestCase { assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y")); assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y")); assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y")); - + assertEquals(true, exec("boolean x = true; def y = true; return x | y")); assertEquals(true, exec("boolean x = true; def y = false; return x | y")); assertEquals(true, exec("boolean x = false; def y = true; return x | y")); assertEquals(false, exec("boolean x = false; def y = false; return x | y")); } - + public void testDefTypedRHS() { expectScriptThrows(ClassCastException.class, () -> { exec("def x = (float)4; byte y = (byte)1; return x | y"); @@ -202,13 +202,13 @@ public class OrTests extends ScriptTestCase { assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y")); assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y")); assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y")); - + assertEquals(true, exec("def x = true; boolean y = true; return x | y")); assertEquals(true, exec("def x = true; boolean y = false; return x | y")); assertEquals(true, exec("def x = false; boolean y = true; return x | y")); assertEquals(false, exec("def x = false; boolean y = false; return x | y")); } - + public void testCompoundAssignment() { // boolean assertEquals(true, exec("boolean x = true; x |= true; return x;")); @@ -231,7 +231,7 @@ public class OrTests extends ScriptTestCase { // long assertEquals((long) (13 | 14), exec("long x = 13L; x |= 14; return x;")); } - + public void testBogusCompoundAssignment() { expectScriptThrows(ClassCastException.class, () -> { exec("float x = 4; int y = 1; x |= y"); @@ -246,7 +246,7 @@ public class OrTests extends ScriptTestCase { exec("int x = 4; double y = 1; x |= y"); }); } - + public void testDefCompoundAssignment() { // boolean assertEquals(true, exec("def x = true; x |= true; return x;")); @@ -269,7 +269,7 @@ public class OrTests extends ScriptTestCase { // long assertEquals((long) (13 | 14), exec("def x = 13L; x |= 14; return x;")); } - + public void testDefBogusCompoundAssignment() { expectScriptThrows(ClassCastException.class, () -> { exec("def x = 4F; int y = 1; x |= y"); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java index a4cfbf2a77c..74e0f90cc1b 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java @@ -44,6 +44,7 @@ import java.util.function.Consumer; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.painless.Definition.DEFINITION; /** * Generates an API reference from the method and type whitelists in {@link Definition}. @@ -67,7 +68,7 @@ public class PainlessDocGenerator { Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE), false, StandardCharsets.UTF_8.name())) { emitGeneratedWarning(indexStream); - List types = Definition.allSimpleTypes().stream().sorted(comparing(t -> t.name)).collect(toList()); + List types = DEFINITION.allSimpleTypes().stream().sorted(comparing(t -> t.name)).collect(toList()); for (Type type : types) { if (type.clazz.isPrimitive()) { // Primitives don't have methods to reference diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index 2e99f652c0a..89159c53647 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -91,7 +91,7 @@ public abstract class ScriptTestCase extends ESTestCase { public Object exec(String script, Map vars, Map compileParams, Scorer scorer, boolean picky) { // test for ambiguity errors before running the actual script if picky is true if (picky) { - Definition definition = Definition.BUILTINS; + Definition definition = Definition.DEFINITION; ScriptClassInfo scriptClassInfo = new ScriptClassInfo(definition, GenericElasticsearchScript.class); CompilerSettings pickySettings = new CompilerSettings(); pickySettings.setPicky(true); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java index ee208991a79..b3842615859 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java @@ -48,7 +48,7 @@ import static org.elasticsearch.painless.node.SSource.MainMethodReserved; * Tests {@link Object#toString} implementations on all extensions of {@link ANode}. */ public class NodeToStringTests extends ESTestCase { - private final Definition definition = Definition.BUILTINS; + private final Definition definition = Definition.DEFINITION; public void testEAssignment() { assertToString( @@ -161,12 +161,12 @@ public class NodeToStringTests extends ESTestCase { public void testECast() { Location l = new Location(getTestName(), 0); AExpression child = new EConstant(l, "test"); - Cast cast = new Cast(Definition.STRING_TYPE, Definition.INT_OBJ_TYPE, true); + Cast cast = new Cast(Definition.DEFINITION.StringType, Definition.DEFINITION.IntegerType, true); assertEquals("(ECast Integer (EConstant String 'test'))", new ECast(l, child, cast).toString()); l = new Location(getTestName(), 1); child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12)); - cast = new Cast(Definition.INT_OBJ_TYPE, Definition.BOOLEAN_OBJ_TYPE, true); + cast = new Cast(Definition.DEFINITION.IntegerType, Definition.DEFINITION.BooleanType, true); assertEquals("(ECast Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))", new ECast(l, child, cast).toString()); } @@ -395,7 +395,7 @@ public class NodeToStringTests extends ESTestCase { public void testPSubBrace() { Location l = new Location(getTestName(), 0); - PSubBrace node = new PSubBrace(l, Definition.INT_TYPE, new ENumeric(l, "1", 10)); + PSubBrace node = new PSubBrace(l, Definition.DEFINITION.intType, new ENumeric(l, "1", 10)); node.prefix = new EVariable(l, "a"); assertEquals("(PSubBrace (EVariable a) (ENumeric 1))", node.toString()); } @@ -761,7 +761,7 @@ public class NodeToStringTests extends ESTestCase { public void testSSubEachArray() { Location l = new Location(getTestName(), 0); - Variable v = new Variable(l, "test", Definition.INT_TYPE, 5, false); + Variable v = new Variable(l, "test", Definition.DEFINITION.intType, 5, false); AExpression e = new ENewArray(l, "int", Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)), true); SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5)))); SSubEachArray node = new SSubEachArray(l, v, e, b); @@ -773,7 +773,7 @@ public class NodeToStringTests extends ESTestCase { public void testSSubEachIterable() { Location l = new Location(getTestName(), 0); - Variable v = new Variable(l, "test", Definition.INT_TYPE, 5, false); + Variable v = new Variable(l, "test", Definition.DEFINITION.intType, 5, false); AExpression e = new EListInit(l, Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3))); SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5)))); SSubEachIterable node = new SSubEachIterable(l, v, e, b); diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 3de4d3f5427..f29daf79912 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -35,6 +35,15 @@ run { setting 'reindex.remote.whitelist', '127.0.0.1:*' } +test { + /* + * We have to disable setting the number of available processors as tests in the + * same JVM randomize processors and will step on each other if we allow them to + * set the number of available processors as it's set-once in Netty. + */ + systemProperty 'es.set.netty.runtime.available.processors', 'false' +} + dependencies { compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" // for http - testing reindex from remote diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index ef02c497a4b..06b683821fd 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -201,7 +201,7 @@ public class TransportReindexAction extends HandledTransportAction header : remoteInfo.getHeaders().entrySet()) { - clientHeaders[i] = new BasicHeader(header.getKey(), header.getValue()); + clientHeaders[i++] = new BasicHeader(header.getKey(), header.getValue()); } return RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme())) .setDefaultHeaders(clientHeaders) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index d7d9cfe051b..c5957ef8be5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -20,17 +20,21 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilderTestCase; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static java.util.Collections.emptyMap; import static java.util.Collections.synchronizedList; import static org.hamcrest.Matchers.hasSize; -public class ReindexFromRemoteBuildRestClientTests extends ESTestCase { +public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase { public void testBuildRestClient() throws Exception { RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); @@ -48,4 +52,22 @@ public class ReindexFromRemoteBuildRestClientTests extends ESTestCase { client.close(); } } + + public void testHeaders() throws Exception { + Map headers = new HashMap<>(); + int numHeaders = randomIntBetween(1, 5); + for (int i = 0; i < numHeaders; ++i) { + headers.put("header" + i, Integer.toString(i)); + } + RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, + headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + long taskId = randomLong(); + List threads = synchronizedList(new ArrayList<>()); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + try { + assertHeaders(client, headers); + } finally { + client.close(); + } + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index 9b19c572c0b..da0dbf2aae3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.Retry; +import org.elasticsearch.client.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; @@ -33,16 +34,17 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Netty4Plugin; import org.junit.After; -import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.concurrent.CyclicBarrier; +import java.util.function.Function; import static java.util.Collections.emptyMap; import static org.elasticsearch.index.reindex.ReindexTestCase.matcher; @@ -51,32 +53,15 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; /** - * Integration test for retry behavior. Useful because retrying relies on the way that the rest of Elasticsearch throws exceptions and unit - * tests won't verify that. + * Integration test for retry behavior. Useful because retrying relies on the way that the + * rest of Elasticsearch throws exceptions and unit tests won't verify that. */ -public class RetryTests extends ESSingleNodeTestCase { +public class RetryTests extends ESIntegTestCase { private static final int DOC_COUNT = 20; private List blockedExecutors = new ArrayList<>(); - - @Before - public void setUp() throws Exception { - super.setUp(); - createIndex("source"); - // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. - BulkRequestBuilder bulk = client().prepareBulk(); - for (int i = 0; i < DOC_COUNT; i++) { - bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); - } - - Retry retry = new Retry(EsRejectedExecutionException.class, BackoffPolicy.exponentialBackoff(), client().threadPool()); - BulkResponse response = retry.withBackoff(client()::bulk, bulk.request(), client().settings()).actionGet(); - assertFalse(response.buildFailureMessage(), response.hasFailures()); - client().admin().indices().prepareRefresh("source").get(); - } - @After public void forceUnblockAllExecutors() { for (CyclicBarrier barrier: blockedExecutors) { @@ -85,8 +70,15 @@ public class RetryTests extends ESSingleNodeTestCase { } @Override - protected Collection> getPlugins() { - return pluginList( + protected Collection> nodePlugins() { + return Arrays.asList( + ReindexPlugin.class, + Netty4Plugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Arrays.asList( ReindexPlugin.class, Netty4Plugin.class); } @@ -95,63 +87,123 @@ public class RetryTests extends ESSingleNodeTestCase { * Lower the queue sizes to be small enough that both bulk and searches will time out and have to be retried. */ @Override - protected Settings nodeSettings() { - Settings.Builder settings = Settings.builder().put(super.nodeSettings()); - // Use pools of size 1 so we can block them - settings.put("thread_pool.bulk.size", 1); - settings.put("thread_pool.search.size", 1); - // Use queues of size 1 because size 0 is broken and because search requests need the queue to function - settings.put("thread_pool.bulk.queue_size", 1); - settings.put("thread_pool.search.queue_size", 1); - // Enable http so we can test retries on reindex from remote. In this case the "remote" cluster is just this cluster. - settings.put(NetworkModule.HTTP_ENABLED.getKey(), true); - // Whitelist reindexing from the http host we're going to use - settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*"); - return settings.build(); + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(nodeSettings()).build(); + } + + final Settings nodeSettings() { + return Settings.builder() + // enable HTTP so we can test retries on reindex from remote; in this case the "remote" cluster is just this cluster + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + // whitelist reindexing from the HTTP host we're going to use + .put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*") + .build(); } public void testReindex() throws Exception { - testCase(ReindexAction.NAME, ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest"), + testCase( + ReindexAction.NAME, + client -> ReindexAction.INSTANCE.newRequestBuilder(client).source("source").destination("dest"), matcher().created(DOC_COUNT)); } public void testReindexFromRemote() throws Exception { - NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0); - TransportAddress address = nodeInfo.getHttp().getAddress().publishAddress(); - RemoteInfo remote = new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, - null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); - ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest") - .setRemoteInfo(remote); - testCase(ReindexAction.NAME, request, matcher().created(DOC_COUNT)); + Function> function = client -> { + /* + * Use the master node for the reindex from remote because that node + * doesn't have a copy of the data on it. + */ + NodeInfo masterNode = null; + for (NodeInfo candidate : client.admin().cluster().prepareNodesInfo().get().getNodes()) { + if (candidate.getNode().isMasterNode()) { + masterNode = candidate; + } + } + assertNotNull(masterNode); + + TransportAddress address = masterNode.getHttp().getAddress().publishAddress(); + RemoteInfo remote = new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, + null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client).source("source").destination("dest") + .setRemoteInfo(remote); + return request; + }; + testCase(ReindexAction.NAME, function, matcher().created(DOC_COUNT)); } public void testUpdateByQuery() throws Exception { - testCase(UpdateByQueryAction.NAME, UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("source"), + testCase(UpdateByQueryAction.NAME, client -> UpdateByQueryAction.INSTANCE.newRequestBuilder(client).source("source"), matcher().updated(DOC_COUNT)); } public void testDeleteByQuery() throws Exception { - testCase(DeleteByQueryAction.NAME, DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("source") + testCase(DeleteByQueryAction.NAME, client -> DeleteByQueryAction.INSTANCE.newRequestBuilder(client).source("source") .filter(QueryBuilders.matchAllQuery()), matcher().deleted(DOC_COUNT)); } - private void testCase(String action, AbstractBulkByScrollRequestBuilder request, BulkIndexByScrollResponseMatcher matcher) + private void testCase( + String action, + Function> request, + BulkIndexByScrollResponseMatcher matcher) throws Exception { - logger.info("Blocking search"); - CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH); + /* + * These test cases work by stuffing the search and bulk queues of a single node and + * making sure that we read and write from that node. Because of some "fun" with the + * way that searches work, we need at least one more node to act as the coordinating + * node for the search request. If we didn't do this then the searches would get stuck + * in the queue anyway because we force queue portions of the coordinating node's + * actions. This is not a big deal in normal operations but a real pain when you are + * intentionally stuffing queues hoping for a failure. + */ + final Settings nodeSettings = Settings.builder() + // use pools of size 1 so we can block them + .put("thread_pool.bulk.size", 1) + .put("thread_pool.search.size", 1) + // use queues of size 1 because size 0 is broken and because search requests need the queue to function + .put("thread_pool.bulk.queue_size", 1) + .put("thread_pool.search.queue_size", 1) + .put("node.attr.color", "blue") + .build(); + final String node = internalCluster().startDataOnlyNode(nodeSettings); + final Settings indexSettings = + Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.routing.allocation.include.color", "blue") + .build(); + + // Create the source index on the node with small thread pools so we can block them. + client().admin().indices().prepareCreate("source").setSettings(indexSettings).execute().actionGet(); + // Not all test cases use the dest index but those that do require that it be on the node will small thread pools + client().admin().indices().prepareCreate("dest").setSettings(indexSettings).execute().actionGet(); + // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. + BulkRequestBuilder bulk = client().prepareBulk(); + for (int i = 0; i < DOC_COUNT; i++) { + bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); + } + + Retry retry = new Retry(EsRejectedExecutionException.class, BackoffPolicy.exponentialBackoff(), client().threadPool()); + BulkResponse initialBulkResponse = retry.withBackoff(client()::bulk, bulk.request(), client().settings()).actionGet(); + assertFalse(initialBulkResponse.buildFailureMessage(), initialBulkResponse.hasFailures()); + client().admin().indices().prepareRefresh("source").get(); + + logger.info("Blocking search"); + CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH, node); + + AbstractBulkByScrollRequestBuilder builder = request.apply(internalCluster().masterClient()); // Make sure we use more than one batch so we have to scroll - request.source().setSize(DOC_COUNT / randomIntBetween(2, 10)); + builder.source().setSize(DOC_COUNT / randomIntBetween(2, 10)); logger.info("Starting request"); - ActionFuture responseListener = request.execute(); + ActionFuture responseListener = builder.execute(); try { logger.info("Waiting for search rejections on the initial search"); assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(0L))); logger.info("Blocking bulk and unblocking search so we start to get bulk rejections"); - CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK); + CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK, node); initialSearchBlock.await(); logger.info("Waiting for bulk rejections"); @@ -161,7 +213,7 @@ public class RetryTests extends ESSingleNodeTestCase { long initialSearchRejections = taskStatus(action).getSearchRetries(); logger.info("Blocking search and unblocking bulk so we should get search rejections for the scroll"); - CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH); + CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH, node); bulkBlock.await(); logger.info("Waiting for search rejections for the scroll"); @@ -187,8 +239,8 @@ public class RetryTests extends ESSingleNodeTestCase { * Blocks the named executor by getting its only thread running a task blocked on a CyclicBarrier and fills the queue with a noop task. * So requests to use this queue should get {@link EsRejectedExecutionException}s. */ - private CyclicBarrier blockExecutor(String name) throws Exception { - ThreadPool threadPool = getInstanceFromNode(ThreadPool.class); + private CyclicBarrier blockExecutor(String name, String node) throws Exception { + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, node); CyclicBarrier barrier = new CyclicBarrier(2); logger.info("Blocking the [{}] executor", name); threadPool.executor(name).execute(() -> { @@ -211,6 +263,11 @@ public class RetryTests extends ESSingleNodeTestCase { * Fetch the status for a task of type "action". Fails if there aren't exactly one of that type of task running. */ private BulkByScrollTask.Status taskStatus(String action) { + /* + * We always use the master client because we always start the test requests on the + * master. We do this simply to make sure that the test request is not started on the + * node who's queue we're manipulating. + */ ListTasksResponse response = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get(); assertThat(response.getTasks(), hasSize(1)); return (BulkByScrollTask.Status) response.getTasks().get(0).getStatus(); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java index f29ca9c10ff..2713f343085 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java @@ -33,7 +33,6 @@ import java.io.IOException; class ByteBufStreamInput extends StreamInput { private final ByteBuf buffer; - private final int startIndex; private final int endIndex; ByteBufStreamInput(ByteBuf buffer, int length) { @@ -41,26 +40,27 @@ class ByteBufStreamInput extends StreamInput { throw new IndexOutOfBoundsException(); } this.buffer = buffer; - startIndex = buffer.readerIndex(); + int startIndex = buffer.readerIndex(); endIndex = startIndex + length; buffer.markReaderIndex(); } @Override public BytesReference readBytesReference(int length) throws IOException { - BytesReference ref = Netty4Utils.toBytesReference(buffer.slice(buffer.readerIndex(), length)); - buffer.skipBytes(length); - return ref; + // NOTE: It is unsafe to share a reference of the internal structure, so we + // use the default implementation which will copy the bytes. It is unsafe because + // a netty ByteBuf might be pooled which requires a manual release to prevent + // memory leaks. + return super.readBytesReference(length); } @Override public BytesRef readBytesRef(int length) throws IOException { - if (!buffer.hasArray()) { - return super.readBytesRef(length); - } - BytesRef bytesRef = new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), length); - buffer.skipBytes(length); - return bytesRef; + // NOTE: It is unsafe to share a reference of the internal structure, so we + // use the default implementation which will copy the bytes. It is unsafe because + // a netty ByteBuf might be pooled which requires a manual release to prevent + // memory leaks. + return super.readBytesRef(length); } @Override diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java index 52dabef7c5d..d02ac2ae2be 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java @@ -19,9 +19,6 @@ package org.elasticsearch.index.analysis; -import java.util.HashSet; -import java.util.List; - import org.apache.commons.codec.Encoder; import org.apache.commons.codec.language.Caverphone1; import org.apache.commons.codec.language.Caverphone2; @@ -45,6 +42,9 @@ import org.elasticsearch.index.analysis.phonetic.HaasePhonetik; import org.elasticsearch.index.analysis.phonetic.KoelnerPhonetik; import org.elasticsearch.index.analysis.phonetic.Nysiis; +import java.util.HashSet; +import java.util.List; + public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { private final Encoder encoder; @@ -116,11 +116,11 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { public TokenStream create(TokenStream tokenStream) { if (encoder == null) { if (ruletype != null && nametype != null) { - if (languageset != null) { - final LanguageSet languages = LanguageSet.from(new HashSet<>(languageset)); - return new BeiderMorseFilter(tokenStream, new PhoneticEngine(nametype, ruletype, true), languages); + LanguageSet langset = null; + if (languageset != null && languageset.size() > 0) { + langset = LanguageSet.from(new HashSet<>(languageset)); } - return new BeiderMorseFilter(tokenStream, new PhoneticEngine(nametype, ruletype, true)); + return new BeiderMorseFilter(tokenStream, new PhoneticEngine(nametype, ruletype, true), langset); } if (maxcodelength > 0) { return new DoubleMetaphoneFilter(tokenStream, maxcodelength, !replace); diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java index 2f84d05563b..e3877faee31 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.analysis; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -26,19 +29,47 @@ import org.elasticsearch.index.Index; import org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin; import org.elasticsearch.test.ESTestCase; import org.hamcrest.MatcherAssert; +import org.junit.Before; import java.io.IOException; +import java.io.StringReader; import static org.hamcrest.Matchers.instanceOf; public class SimplePhoneticAnalysisTests extends ESTestCase { - public void testPhoneticTokenFilterFactory() throws IOException { + + private TestAnalysis analysis; + + @Before + public void setup() throws IOException { String yaml = "/org/elasticsearch/index/analysis/phonetic-1.yml"; Settings settings = Settings.builder().loadFromStream(yaml, getClass().getResourceAsStream(yaml), false) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin()); + this.analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin()); + } + + public void testPhoneticTokenFilterFactory() throws IOException { TokenFilterFactory filterFactory = analysis.tokenFilter.get("phonetic"); MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class)); } + + public void testPhoneticTokenFilterBeiderMorseNoLanguage() throws IOException { + TokenFilterFactory filterFactory = analysis.tokenFilter.get("beidermorsefilter"); + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader("ABADIAS")); + String[] expected = new String[] { "abYdias", "abYdios", "abadia", "abadiaS", "abadias", "abadio", "abadioS", "abadios", "abodia", + "abodiaS", "abodias", "abodio", "abodioS", "abodios", "avadias", "avadios", "avodias", "avodios", "obadia", "obadiaS", + "obadias", "obadio", "obadioS", "obadios", "obodia", "obodiaS", "obodias", "obodioS" }; + BaseTokenStreamTestCase.assertTokenStreamContents(filterFactory.create(tokenizer), expected); + } + + public void testPhoneticTokenFilterBeiderMorseWithLanguage() throws IOException { + TokenFilterFactory filterFactory = analysis.tokenFilter.get("beidermorsefilterfrench"); + Tokenizer tokenizer = new WhitespaceTokenizer(); + tokenizer.setReader(new StringReader("Rimbault")); + String[] expected = new String[] { "rimbD", "rimbDlt", "rimba", "rimbalt", "rimbo", "rimbolt", "rimbu", "rimbult", "rmbD", "rmbDlt", + "rmba", "rmbalt", "rmbo", "rmbolt", "rmbu", "rmbult" }; + BaseTokenStreamTestCase.assertTokenStreamContents(filterFactory.create(tokenizer), expected); + } } diff --git a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml index 6c0a0763881..1909c7ee063 100644 --- a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml +++ b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml @@ -19,6 +19,10 @@ index: beidermorsefilter: type: phonetic encoder: beidermorse + beidermorsefilterfrench: + type: phonetic + encoder: beidermorse + languageset : [ "french" ] koelnerphonetikfilter: type: phonetic encoder: koelnerphonetik diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 8da6d8d3afe..f4e2f0cb7b0 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -31,7 +31,9 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystemException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFilePermission; @@ -40,8 +42,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; /** * Create a simple "daemon controller", put it in the right place and check that it runs. @@ -189,6 +194,33 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase { equalTo("plugin [test_plugin] does not have permission to fork native controller")); } + public void testSpawnerHandlingOfDesktopServicesStoreFiles() throws IOException { + final Path esHome = createTempDir().resolve("home"); + final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString()).build(); + + final Environment environment = new Environment(settings); + + Files.createDirectories(environment.pluginsFile()); + + final Path desktopServicesStore = environment.pluginsFile().resolve(".DS_Store"); + Files.createFile(desktopServicesStore); + + final Spawner spawner = new Spawner(); + if (Constants.MAC_OS_X) { + // if the spawner were not skipping the Desktop Services Store files on macOS this would explode + spawner.spawnNativePluginControllers(environment); + } else { + // we do not ignore these files on non-macOS systems + final FileSystemException e = + expectThrows(FileSystemException.class, () -> spawner.spawnNativePluginControllers(environment)); + if (Constants.WINDOWS) { + assertThat(e, instanceOf(NoSuchFileException.class)); + } else { + assertThat(e, hasToString(containsString("Not a directory"))); + } + } + } + private void createControllerProgram(final Path outputFile) throws IOException { final Path outputDir = outputFile.getParent(); Files.createDirectories(outputDir); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json index 96c4c53cd9d..3fb9d1e207e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json @@ -8,7 +8,7 @@ "parts": { "name": { "type": "list", - "required": false, + "required": true, "description": "The comma separated names of the index templates" } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml index e2b15f3bde2..3e8b3db468e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml @@ -210,3 +210,16 @@ catch: missing indices.get_template: name: "my_template" + +--- +"Put index template without index_patterns": + + - skip: + version: " - 5.99.99" + reason: the error message is updated in v6.0.0 + + - do: + catch: /index patterns are missing/ + indices.put_template: + name: test + body: {} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yml index 2617f76941c..ad8058876ae 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yml @@ -1,5 +1,8 @@ --- "Discovery stats": + - skip: + version: " - 6.0.99" + reason: "published_cluster_states_received arrived in 6.1.0" - do: cluster.state: {} @@ -15,6 +18,11 @@ - is_true: nodes.$master.name - is_false: nodes.$master.jvm - is_true: nodes.$master.discovery + - is_true: nodes.$master.discovery.cluster_state_queue + - is_true: nodes.$master.discovery.published_cluster_states + - gte: { nodes.$master.discovery.published_cluster_states.full_states: 0 } + - gte: { nodes.$master.discovery.published_cluster_states.incompatible_diffs: 0 } + - gte: { nodes.$master.discovery.published_cluster_states.compatible_diffs: 0 } - is_true: nodes.$master.roles - do: @@ -26,4 +34,9 @@ - is_false: nodes.$master.name - is_false: nodes.$master.jvm - is_true: nodes.$master.discovery + - is_true: nodes.$master.discovery.cluster_state_queue + - is_true: nodes.$master.discovery.published_cluster_states + - gte: { nodes.$master.discovery.published_cluster_states.full_states: 0 } + - gte: { nodes.$master.discovery.published_cluster_states.incompatible_diffs: 0 } + - gte: { nodes.$master.discovery.published_cluster_states.compatible_diffs: 0 } - is_false: nodes.$master.roles diff --git a/test/framework/src/main/java/org/elasticsearch/client/RestClientBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/client/RestClientBuilderTestCase.java new file mode 100644 index 00000000000..086dca0d949 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/client/RestClientBuilderTestCase.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.HashMap; +import java.util.Map; + +import joptsimple.internal.Strings; +import org.apache.http.Header; +import org.elasticsearch.test.ESTestCase; + +/** + * A test case with access to internals of a RestClient. + */ +public abstract class RestClientBuilderTestCase extends ESTestCase { + /** Checks the given rest client has the provided default headers. */ + public void assertHeaders(RestClient client, Map expectedHeaders) { + expectedHeaders = new HashMap<>(expectedHeaders); // copy so we can remove as we check + for (Header header : client.defaultHeaders) { + String name = header.getName(); + String expectedValue = expectedHeaders.remove(name); + if (expectedValue == null) { + fail("Found unexpected header in rest client: " + name); + } + assertEquals(expectedValue, header.getValue()); + } + if (expectedHeaders.isEmpty() == false) { + fail("Missing expected headers in rest client: " + Strings.join(expectedHeaders.keySet(), ", ")); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 4f6e8bb1924..dd8f90b9e71 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -67,7 +67,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService { null, null, null, null, null, fsInfo, null, null, null, - null, null, null); + null, null, null, null); } public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client, diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 243ab11e61f..121471bdb07 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -765,8 +765,8 @@ public abstract class ESTestCase extends LuceneTestCase { return terminated; } - public static boolean terminate(ThreadPool service) throws InterruptedException { - return ThreadPool.terminate(service, 10, TimeUnit.SECONDS); + public static boolean terminate(ThreadPool threadPool) throws InterruptedException { + return ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index c8eec0cbb54..9ad0afdf6a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2069,7 +2069,8 @@ public final class InternalTestCluster extends TestCluster { NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node); CommonStatsFlags flags = new CommonStatsFlags(Flag.FieldData, Flag.QueryCache, Flag.Segments); - NodeStats stats = nodeService.stats(flags, false, false, false, false, false, false, false, false, false, false, false); + NodeStats stats = nodeService.stats(flags, + false, false, false, false, false, false, false, false, false, false, false, false); assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L)); assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0L)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index 152af0c85f6..b4704bd9ed8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.rest.yaml; import com.carrotsearch.randomizedtesting.RandomizedTest; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -42,6 +43,9 @@ import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; /** * Used by {@link ESClientYamlSuiteTestCase} to execute REST requests according to the tests written in yaml suite files. Wraps a @@ -80,24 +84,40 @@ public class ClientYamlTestClient { //divide params between ones that go within query string and ones that go within path Map pathParts = new HashMap<>(); Map queryStringParams = new HashMap<>(); + + Set apiRequiredPathParts = restApi.getPathParts().entrySet().stream().filter(e -> e.getValue() == true).map(Entry::getKey) + .collect(Collectors.toSet()); + Set apiRequiredParameters = restApi.getParams().entrySet().stream().filter(e -> e.getValue() == true).map(Entry::getKey) + .collect(Collectors.toSet()); + for (Map.Entry entry : params.entrySet()) { - if (restApi.getPathParts().contains(entry.getKey())) { + if (restApi.getPathParts().containsKey(entry.getKey())) { pathParts.put(entry.getKey(), entry.getValue()); + apiRequiredPathParts.remove(entry.getKey()); + } else if (restApi.getParams().containsKey(entry.getKey()) + || restSpec.isGlobalParameter(entry.getKey()) + || restSpec.isClientParameter(entry.getKey())) { + queryStringParams.put(entry.getKey(), entry.getValue()); + apiRequiredParameters.remove(entry.getKey()); } else { - if (restApi.getParams().contains(entry.getKey()) || restSpec.isGlobalParameter(entry.getKey()) - || restSpec.isClientParameter(entry.getKey())) { - queryStringParams.put(entry.getKey(), entry.getValue()); - } else { - throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in [" - + restApi.getName() + "] " + "api"); - } + throw new IllegalArgumentException( + "path/param [" + entry.getKey() + "] not supported by [" + restApi.getName() + "] " + "api"); } } + if (false == apiRequiredPathParts.isEmpty()) { + throw new IllegalArgumentException( + "missing required path part: " + apiRequiredPathParts + " by [" + restApi.getName() + "] api"); + } + if (false == apiRequiredParameters.isEmpty()) { + throw new IllegalArgumentException( + "missing required parameter: " + apiRequiredParameters + " by [" + restApi.getName() + "] api"); + } + List supportedMethods = restApi.getSupportedMethods(pathParts.keySet()); String requestMethod; if (entity != null) { - if (!restApi.isBodySupported()) { + if (false == restApi.isBodySupported()) { throw new IllegalArgumentException("body is not supported by [" + restApi.getName() + "] api"); } String contentType = entity.getContentType().getValue(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java index 18784f5ce9a..72c94762a43 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java @@ -22,6 +22,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -35,8 +36,8 @@ public class ClientYamlSuiteRestApi { private final String name; private List methods = new ArrayList<>(); private List paths = new ArrayList<>(); - private List pathParts = new ArrayList<>(); - private List params = new ArrayList<>(); + private Map pathParts = new HashMap<>(); + private Map params = new HashMap<>(); private Body body = Body.NOT_SUPPORTED; public enum Body { @@ -98,20 +99,28 @@ public class ClientYamlSuiteRestApi { this.paths.add(path); } - public List getPathParts() { + /** + * Gets all path parts supported by the api. For every path part defines if it + * is required or optional. + */ + public Map getPathParts() { return pathParts; } - void addPathPart(String pathPart) { - this.pathParts.add(pathPart); + void addPathPart(String pathPart, boolean required) { + this.pathParts.put(pathPart, required); } - public List getParams() { + /** + * Gets all parameters supported by the api. For every parameter defines if it + * is required or optional. + */ + public Map getParams() { return params; } - void addParam(String param) { - this.params.add(param); + void addParam(String param, boolean required) { + this.params.put(param, required); } void setBodyOptional() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java index 8abcfc35f27..66dc0a57054 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.test.rest.yaml.restspec; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -27,6 +29,11 @@ import java.io.IOException; */ public class ClientYamlSuiteRestApiParser { + private static final ObjectParser PARAMETER_PARSER = new ObjectParser<>("parameter", true, Parameter::new); + static { + PARAMETER_PARSER.declareBoolean(Parameter::setRequired, new ParseField("required")); + } + public ClientYamlSuiteRestApi parse(String location, XContentParser parser) throws IOException { while ( parser.nextToken() != XContentParser.Token.FIELD_NAME ) { @@ -57,7 +64,6 @@ public class ClientYamlSuiteRestApiParser { if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } - if (parser.currentToken() == XContentParser.Token.START_ARRAY && "paths".equals(currentFieldName)) { while (parser.nextToken() == XContentParser.Token.VALUE_STRING) { String path = parser.text(); @@ -71,30 +77,30 @@ public class ClientYamlSuiteRestApiParser { if (parser.currentToken() == XContentParser.Token.START_OBJECT && "parts".equals(currentFieldName)) { while (parser.nextToken() == XContentParser.Token.FIELD_NAME) { String part = parser.currentName(); - if (restApi.getPathParts().contains(part)) { + if (restApi.getPathParts().containsKey(part)) { throw new IllegalArgumentException("Found duplicate part [" + part + "]"); } - restApi.addPathPart(part); parser.nextToken(); if (parser.currentToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Expected parts field in rest api definition to contain an object"); } - parser.skipChildren(); + restApi.addPathPart(part, PARAMETER_PARSER.parse(parser, null).isRequired()); } } if (parser.currentToken() == XContentParser.Token.START_OBJECT && "params".equals(currentFieldName)) { while (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + String param = parser.currentName(); - if (restApi.getParams().contains(param)) { + if (restApi.getParams().containsKey(param)) { throw new IllegalArgumentException("Found duplicate param [" + param + "]"); } - restApi.addParam(parser.currentName()); + parser.nextToken(); if (parser.currentToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Expected params field in rest api definition to contain an object"); } - parser.skipChildren(); + restApi.addParam(param, PARAMETER_PARSER.parse(parser, null).isRequired()); } } @@ -124,7 +130,7 @@ public class ClientYamlSuiteRestApiParser { } } } - if (!requiredFound) { + if (false == requiredFound) { restApi.setBodyOptional(); } } @@ -146,4 +152,14 @@ public class ClientYamlSuiteRestApiParser { return restApi; } + + private static class Parameter { + private boolean required; + public boolean isRequired() { + return required; + } + public void setRequired(boolean required) { + this.required = required; + } + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java index 6acdd935400..ddf89f9a6fc 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java @@ -22,7 +22,9 @@ import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.rest.yaml.section.AbstractClientYamlTestFragmentParserTestCase; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.notNullValue; public class ClientYamlSuiteRestApiParserTests extends AbstractClientYamlTestFragmentParserTestCase { @@ -39,11 +41,13 @@ public class ClientYamlSuiteRestApiParserTests extends AbstractClientYamlTestFra assertThat(restApi.getPaths().get(0), equalTo("/{index}/{type}")); assertThat(restApi.getPaths().get(1), equalTo("/{index}/{type}/{id}")); assertThat(restApi.getPathParts().size(), equalTo(3)); - assertThat(restApi.getPathParts().get(0), equalTo("id")); - assertThat(restApi.getPathParts().get(1), equalTo("index")); - assertThat(restApi.getPathParts().get(2), equalTo("type")); + assertThat(restApi.getPathParts().keySet(), containsInAnyOrder("id", "index", "type")); + assertThat(restApi.getPathParts(), hasEntry("index", true)); + assertThat(restApi.getPathParts(), hasEntry("type", true)); + assertThat(restApi.getPathParts(), hasEntry("id", false)); assertThat(restApi.getParams().size(), equalTo(4)); - assertThat(restApi.getParams(), contains("wait_for_active_shards", "op_type", "parent", "refresh")); + assertThat(restApi.getParams().keySet(), containsInAnyOrder("wait_for_active_shards", "op_type", "parent", "refresh")); + restApi.getParams().entrySet().stream().forEach(e -> assertThat(e.getValue(), equalTo(false))); assertThat(restApi.isBodySupported(), equalTo(true)); assertThat(restApi.isBodyRequired(), equalTo(true)); } @@ -59,7 +63,7 @@ public class ClientYamlSuiteRestApiParserTests extends AbstractClientYamlTestFra assertThat(restApi.getPaths().get(0), equalTo("/_template")); assertThat(restApi.getPaths().get(1), equalTo("/_template/{name}")); assertThat(restApi.getPathParts().size(), equalTo(1)); - assertThat(restApi.getPathParts().get(0), equalTo("name")); + assertThat(restApi.getPathParts(), hasEntry("name", false)); assertThat(restApi.getParams().size(), equalTo(0)); assertThat(restApi.isBodySupported(), equalTo(false)); assertThat(restApi.isBodyRequired(), equalTo(false)); @@ -78,10 +82,11 @@ public class ClientYamlSuiteRestApiParserTests extends AbstractClientYamlTestFra assertThat(restApi.getPaths().get(1), equalTo("/{index}/_count")); assertThat(restApi.getPaths().get(2), equalTo("/{index}/{type}/_count")); assertThat(restApi.getPathParts().size(), equalTo(2)); - assertThat(restApi.getPathParts().get(0), equalTo("index")); - assertThat(restApi.getPathParts().get(1), equalTo("type")); + assertThat(restApi.getPathParts().keySet(), containsInAnyOrder("index", "type")); + restApi.getPathParts().entrySet().stream().forEach(e -> assertThat(e.getValue(), equalTo(false))); assertThat(restApi.getParams().size(), equalTo(1)); - assertThat(restApi.getParams().get(0), equalTo("ignore_unavailable")); + assertThat(restApi.getParams().keySet(), contains("ignore_unavailable")); + assertThat(restApi.getParams(), hasEntry("ignore_unavailable", false)); assertThat(restApi.isBodySupported(), equalTo(true)); assertThat(restApi.isBodyRequired(), equalTo(false)); }