From 5c1d3aa2f099a313a76d1c2cf6702856f43dcc32 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 5 Feb 2018 11:57:21 -0800 Subject: [PATCH 01/13] Painless: Fixes a null pointer exception in certain cases of for loop usage (#28506) The initializer and afterthought were not having their types appropriately cast which is necessary with expressions which in turn caused values to be popped off the stack that were null. --- .../main/java/org/elasticsearch/painless/node/SFor.java | 9 ++++++++- .../org/elasticsearch/painless/BasicStatementTests.java | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java index a0b3edf6591..c12ad5473f9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java @@ -76,7 +76,7 @@ public final class SFor extends AStatement { locals = Locals.newLocalScope(locals); if (initializer != null) { - if (initializer instanceof AStatement) { + if (initializer instanceof SDeclBlock) { initializer.analyze(locals); } else if (initializer instanceof AExpression) { AExpression initializer = (AExpression)this.initializer; @@ -87,6 +87,9 @@ public final class SFor extends AStatement { if (!initializer.statement) { throw createError(new IllegalArgumentException("Not a statement.")); } + + initializer.expected = initializer.actual; + this.initializer = initializer.cast(locals); } else { throw createError(new IllegalStateException("Illegal tree structure.")); } @@ -119,6 +122,9 @@ public final class SFor extends AStatement { if (!afterthought.statement) { throw createError(new IllegalArgumentException("Not a statement.")); } + + afterthought.expected = afterthought.actual; + afterthought = afterthought.cast(locals); } if (block != null) { @@ -197,6 +203,7 @@ public final class SFor extends AStatement { if (afterthought != null) { writer.mark(begin); afterthought.write(writer, globals); + writer.writePop(MethodWriter.getType(afterthought.expected).getSize()); } if (afterthought != null || !allEscape) { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java index fc2fffb6441..e397013e06f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java @@ -108,8 +108,12 @@ public class BasicStatementTests extends ScriptTestCase { } public void testForStatement() { + assertEquals(6, exec("int x, y; for (x = 0; x < 4; ++x) {y += x;} return y;")); assertEquals("aaaaaa", exec("String c = \"a\"; for (int x = 0; x < 5; ++x) c += \"a\"; return c;")); + assertEquals(6, exec("double test() { return 0.0; }" + + "int x, y; for (test(); x < 4; test()) {y += x; ++x;} return y;")); + Object value = exec( " int[][] b = new int[5][5]; \n" + " for (int x = 0; x < 5; ++x) { \n" + From eebff4d2b3793b2e9f93ffe63ec015211fca5321 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 5 Feb 2018 16:18:18 -0700 Subject: [PATCH 02/13] Use non deprecated xcontenthelper (#28503) * Move to non-deprecated XContentHelper.createParser(...) This moves away from one of the now-deprecated XContentHelper.createParser methods in favor of specifying the deprecation logger at parser creation time. Relates to #28449 Note that this doesn't move all the `createParser` calls because some of them use the already-deprecated method that doesn't specify the XContentType. * Remove the deprecated (and now non-needed) createParser method --- .../main/java/org/elasticsearch/client/Request.java | 4 +++- .../elasticsearch/client/RestHighLevelClient.java | 4 +++- .../ingest/common/ScriptProcessor.java | 4 +++- .../index/rankeval/TransportRankEvalAction.java | 4 +++- .../reindex/remote/RemoteScrollableHitSource.java | 3 ++- .../ingest/useragent/UserAgentParser.java | 4 +++- .../action/admin/indices/alias/Alias.java | 8 ++++---- .../elasticsearch/action/update/UpdateRequest.java | 7 +++++-- .../common/xcontent/XContentHelper.java | 12 ++++++------ .../elasticsearch/index/mapper/DocumentParser.java | 4 +++- .../repositories/blobstore/BlobStoreRepository.java | 7 +++++-- .../completion/CompletionSuggestionBuilder.java | 4 +++- .../metadata/IndexTemplateMetaDataTests.java | 13 +++++++++---- .../org/elasticsearch/test/XContentTestUtils.java | 4 +++- .../rest/yaml/restspec/ClientYamlSuiteRestSpec.java | 4 +++- .../test/rest/yaml/section/ClientYamlTestSuite.java | 4 +++- .../elasticsearch/test/XContentTestUtilsTests.java | 7 +++++-- 17 files changed, 66 insertions(+), 31 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java index 3061c67cd5c..4ec7315b73b 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java @@ -58,6 +58,7 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContent; @@ -316,7 +317,8 @@ public final class Request { BytesReference indexSource = indexRequest.source(); XContentType indexXContentType = indexRequest.getContentType(); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, indexSource, indexXContentType)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) { try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) { builder.copyCurrentStructure(parser); source = builder.bytes().toBytesRef(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index c8e248657dd..5d5f0e145ef 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -51,6 +51,7 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -637,7 +638,8 @@ public class RestHighLevelClient implements Closeable { if (xContentType == null) { throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue()); } - try (XContentParser parser = xContentType.xContent().createParser(registry, entity.getContent())) { + try (XContentParser parser = xContentType.xContent().createParser(registry, + LoggingDeprecationHandler.INSTANCE, entity.getContent())) { return entityParser.apply(parser); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index 92bcd8f9b1e..ac8c1e3528a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -21,6 +21,7 @@ package org.elasticsearch.ingest.common; import com.fasterxml.jackson.core.JsonFactory; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -97,7 +98,8 @@ public final class ScriptProcessor extends AbstractProcessor { public ScriptProcessor create(Map registry, String processorTag, Map config) throws Exception { XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); - XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, builder.bytes().streamInput()); + XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, builder.bytes().streamInput()); Script script = Script.parse(parser); Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 7299869bcf8..b677f663998 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -105,7 +106,8 @@ public class TransportRankEvalAction extends HandledTransportAction filter = parser.mapOrdered(); alias.filter(filter); } } else if (token == XContentParser.Token.VALUE_STRING) { - if (ROUTING.match(currentFieldName)) { + if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { alias.routing(parser.text()); - } else if (INDEX_ROUTING.match(currentFieldName)) { + } else if (INDEX_ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { alias.indexRouting(parser.text()); - } else if (SEARCH_ROUTING.match(currentFieldName)) { + } else if (SEARCH_ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { alias.searchRouting(parser.text()); } } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 9bfb78f5058..2dcd35dfb36 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -864,7 +865,8 @@ public class UpdateRequest extends InstanceShardOperationRequest } if (doc != null) { XContentType xContentType = doc.getContentType(); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, doc.source(), xContentType)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, doc.source(), xContentType)) { builder.field("doc"); builder.copyCurrentStructure(parser); } @@ -874,7 +876,8 @@ public class UpdateRequest extends InstanceShardOperationRequest } if (upsertRequest != null) { XContentType xContentType = upsertRequest.getContentType(); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, upsertRequest.source(), xContentType)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, upsertRequest.source(), xContentType)) { builder.field("upsert"); builder.copyCurrentStructure(parser); } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index fc58e8f58a7..76d8aa15f87 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -40,7 +40,7 @@ public class XContentHelper { /** * Creates a parser based on the bytes provided - * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection + * @deprecated use {@link #createParser(NamedXContentRegistry, DeprecationHandler, BytesReference, XContentType)} to avoid content type auto-detection */ @Deprecated public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException { @@ -60,8 +60,8 @@ public class XContentHelper { /** * Creates a parser for the bytes using the supplied content-type */ - public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes, - XContentType xContentType) throws IOException { + public static XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, + BytesReference bytes, XContentType xContentType) throws IOException { Objects.requireNonNull(xContentType); Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { @@ -69,7 +69,7 @@ public class XContentHelper { if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput); + return XContentFactory.xContent(xContentType).createParser(xContentRegistry, deprecationHandler, compressedInput); } else { return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput()); } @@ -131,7 +131,7 @@ public class XContentHelper { public static Map convertToMap(XContent xContent, InputStream input, boolean ordered) throws ElasticsearchParseException { // It is safe to use EMPTY here because this never uses namedObject - try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) { + try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, input)) { return ordered ? parser.mapOrdered() : parser.map(); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); @@ -161,7 +161,7 @@ public class XContentHelper { // It is safe to use EMPTY here because this never uses namedObject try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY, - bytes.streamInput())) { + LoggingDeprecationHandler.INSTANCE, bytes.streamInput())) { parser.nextToken(); XContentBuilder builder = XContentFactory.jsonBuilder(); if (prettyPrint) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index aa286b88346..ae800529948 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.joda.FormatDateTimeFormatter; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -61,7 +62,8 @@ final class DocumentParser { final ParseContext.InternalParseContext context; final XContentType xContentType = source.getXContentType(); - try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), source.source(), xContentType)) { + try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), + LoggingDeprecationHandler.INSTANCE, source.source(), xContentType)) { context = new ParseContext.InternalParseContext(indexSettings.getSettings(), docMapperParser, docMapper, source, parser); validateStart(parser); internalParseDocument(mapping, context, parser); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 9068c6ff397..330b2d29986 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -68,6 +68,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -605,7 +606,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); // EMPTY is safe here because RepositoryData#fromXContent calls namedObject - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, out.bytes(), XContentType.JSON)) { repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen); } catch (NotXContentException e) { logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length()); @@ -617,7 +619,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, out.bytes(), XContentType.JSON)) { repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser); } } catch (NoSuchFileException e) { diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java index 224204bfc8d..97dcd27b5d9 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; @@ -311,7 +312,8 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder> parseContextBytes(BytesReference contextBytes, NamedXContentRegistry xContentRegistry, ContextMappings contextMappings) throws IOException { - try (XContentParser contextParser = XContentHelper.createParser(xContentRegistry, contextBytes, CONTEXT_BYTES_XCONTENT_TYPE)) { + try (XContentParser contextParser = XContentHelper.createParser(xContentRegistry, + LoggingDeprecationHandler.INSTANCE, contextBytes, CONTEXT_BYTES_XCONTENT_TYPE)) { contextParser.nextToken(); Map> queryContexts = new HashMap<>(contextMappings.size()); assert contextParser.currentToken() == XContentParser.Token.START_OBJECT; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java index d5f441436e7..0d2443c4202 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -102,7 +103,8 @@ public class IndexTemplateMetaDataTests extends ESTestCase { BytesReference templateBytes = new BytesArray(template); final IndexTemplateMetaData indexTemplateMetaData; - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytes, XContentType.JSON)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytes, XContentType.JSON)) { indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test"); } @@ -115,7 +117,8 @@ public class IndexTemplateMetaDataTests extends ESTestCase { } final IndexTemplateMetaData indexTemplateMetaDataRoundTrip; - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytesRoundTrip, XContentType.JSON)) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytesRoundTrip, XContentType.JSON)) { indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test"); } assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip)); @@ -142,7 +145,8 @@ public class IndexTemplateMetaDataTests extends ESTestCase { randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" + "}}}"; try (XContentParser parser = - XContentHelper.createParser(NamedXContentRegistry.EMPTY, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) { + XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) { final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100))); assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []")); @@ -156,7 +160,8 @@ public class IndexTemplateMetaDataTests extends ESTestCase { randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" + "}}}"; try (XContentParser parser = - XContentHelper.createParser(NamedXContentRegistry.EMPTY, new BytesArray(templateWithoutPattern), XContentType.JSON)) { + XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithoutPattern), XContentType.JSON)) { final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100))); assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got null")); diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index c8a0ce8fc28..063f0554443 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -20,6 +20,7 @@ package org.elasticsearch.test; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContent; @@ -184,7 +185,8 @@ public final class XContentTestUtils { List insertPaths; // we can use NamedXContentRegistry.EMPTY here because we only traverse the xContent once and don't use it - try (XContentParser parser = createParser(NamedXContentRegistry.EMPTY, xContent, contentType)) { + try (XContentParser parser = createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, xContent, contentType)) { parser.nextToken(); List possiblePaths = XContentTestUtils.getInsertPaths(parser, new Stack<>()); if (excludeFilter == null) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java index dd800f5c9dd..70665ad5d9b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.stream.Stream; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -94,7 +95,8 @@ public class ClientYamlSuiteRestSpec { private static void parseSpecFile(ClientYamlSuiteRestApiParser restApiParser, Path jsonFile, ClientYamlSuiteRestSpec restSpec) { try (InputStream stream = Files.newInputStream(jsonFile)) { - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, stream)) { + try (XContentParser parser = + JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { String filename = jsonFile.getFileName().toString(); if (filename.equals("_common.json")) { String currentFieldName = null; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index eab62c25145..72c83f632ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -19,6 +19,8 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.yaml.YamlXContent; @@ -63,7 +65,7 @@ public class ClientYamlTestSuite { } try (XContentParser parser = YamlXContent.yamlXContent.createParser(ExecutableSection.XCONTENT_REGISTRY, - Files.newInputStream(file))) { + LoggingDeprecationHandler.INSTANCE, Files.newInputStream(file))) { return parse(api, filename, parser); } catch(Exception e) { throw new IOException("Error parsing " + api + "/" + filename, e); diff --git a/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java index f3b44f25104..4d03e8e1bb0 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/XContentTestUtilsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -71,7 +72,8 @@ public class XContentTestUtilsTests extends ESTestCase { } builder.endObject(); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, builder.bytes(), builder.contentType())) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, builder.bytes(), builder.contentType())) { parser.nextToken(); List insertPaths = XContentTestUtils.getInsertPaths(parser, new Stack<>()); assertEquals(5, insertPaths.size()); @@ -96,7 +98,8 @@ public class XContentTestUtilsTests extends ESTestCase { Collections.singletonList("inn\\.er1"), () -> "inner2", () -> new HashMap<>()); builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), builder.bytes(), Collections.singletonList("inn\\.er1"), () -> "field2", () -> "value2"); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, builder.bytes(), builder.contentType())) { + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, builder.bytes(), builder.contentType())) { Map map = parser.map(); assertEquals(2, map.size()); assertEquals("value1", map.get("field1")); From 38c5f4efeec84ce021e012fc47913ffa1f32bc05 Mon Sep 17 00:00:00 2001 From: Ivan Brusic Date: Tue, 6 Feb 2018 00:25:28 -0800 Subject: [PATCH 03/13] =?UTF-8?q?[Docs]=C2=A0Fix=20incomplete=20URLs=20(#2?= =?UTF-8?q?8528)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugins/authors.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc index 8d5de86c864..059d9a5bf68 100644 --- a/docs/plugins/authors.asciidoc +++ b/docs/plugins/authors.asciidoc @@ -5,9 +5,9 @@ The Elasticsearch repository contains examples of: -* a https://github.com/elastic/elasticsearch/tree/master/plugins/custom-settings[Java plugin] +* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/custom-settings[Java plugin] which contains a plugin with custom settings. -* a https://github.com/elastic/elasticsearch/tree/master/plugins/rest-handler[Java plugin] +* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/rest-handler[Java plugin] which contains a plugin that registers a Rest handler. * a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/rescore[Java plugin] which contains a rescore plugin. From 2a35b4ee2b9253dcb51d3e205b605604a637a2f2 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 6 Feb 2018 11:26:48 +0100 Subject: [PATCH 04/13] Use right skip versions. Closes #27570 --- .../test/search.aggregation/200_top_hits_metric.yml | 4 ++-- .../rest-api-spec/test/search.inner_hits/10_basic.yml | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index f48aee28dc2..8d53322274b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -1,8 +1,8 @@ --- "top_hits aggregation with nested documents": - skip: - version: "5.99.99 - " - reason: "5.x nodes don't include index or id in nested top hits" + version: " - 6.1.99" + reason: "<= 6.1 nodes don't always include index or id in nested top hits" - do: indices.create: index: my-index diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index 80d3f924d2b..40e9d705ea4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -13,9 +13,8 @@ setup: --- "Nested inner hits": - skip: - # this will only run in a mixed cluster environment with at least 1 5.x node - version: "5.99.99 - " - reason: multiple types are not supported on 6.x indices onwards + version: " - 6.1.99" + reason: "<= 6.1 nodes don't always include index or id in nested inner hits" - do: index: index: test @@ -35,7 +34,7 @@ setup: - match: { hits.hits.0._type: "type_1" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._index: "test" } - - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "type1" } + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "type_1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.field: "nested_field" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 } From d43f0b5f26018c8a3e19062dae6d36d885c191be Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 6 Feb 2018 14:24:52 +0100 Subject: [PATCH 05/13] Improve failure message when restoring an index that already exists in the cluster (#28498) Makes the message more actionable and removes the focus on the fact that the index is open. --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 63b461afbd7..0a929cc8f0b 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -362,7 +362,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp // Index exist - checking that it's closed if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) { // TODO: Enable restore for open indices - throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because it's open"); + throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because an open index with same name already exists in the cluster. " + + "Either close or delete the existing index or restore the index under a different name by providing a rename pattern and replacement name"); } // Index exist - checking if it's partial restore if (partial) { From c8df4460008ede0b53372824f84229f1d040c3e2 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 6 Feb 2018 17:29:22 +0100 Subject: [PATCH 06/13] No refresh on shard activation needed (#28013) A shard is fully baked when it moves to POST_RECOVERY. There is no need to do an extra refresh on shard activation again as the shard has already been refreshed when it moved to POST_RECOVERY. --- .../main/java/org/elasticsearch/index/shard/IndexShard.java | 6 ------ .../java/org/elasticsearch/index/shard/IndexShardTests.java | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 5245da668de..ced906d687c 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -407,12 +407,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl if (state == IndexShardState.POST_RECOVERY && newRouting.active()) { assert currentRouting.active() == false : "we are in POST_RECOVERY, but our shard routing is active " + currentRouting; - // we want to refresh *before* we move to internal STARTED state - try { - getEngine().refresh("cluster_state_started"); - } catch (Exception e) { - logger.debug("failed to refresh due to move to cluster wide started", e); - } if (newRouting.primary() && currentRouting.isRelocationTarget() == false) { replicationTracker.activatePrimaryMode(getEngine().getLocalCheckpointTracker().getCheckpoint()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 9d854331b90..77f6149c4a3 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1170,14 +1170,14 @@ public class IndexShardTests extends IndexShardTestCase { public void testRefreshMetric() throws IOException { IndexShard shard = newStartedShard(); - assertThat(shard.refreshStats().getTotal(), equalTo(3L)); // refresh on: finalize, end of recovery and on starting shard + assertThat(shard.refreshStats().getTotal(), equalTo(2L)); // refresh on: finalize and end of recovery long initialTotalTime = shard.refreshStats().getTotalTimeInMillis(); // check time advances for (int i = 1; shard.refreshStats().getTotalTimeInMillis() == initialTotalTime; i++) { indexDoc(shard, "test", "test"); - assertThat(shard.refreshStats().getTotal(), equalTo(3L + i - 1)); + assertThat(shard.refreshStats().getTotal(), equalTo(2L + i - 1)); shard.refresh("test"); - assertThat(shard.refreshStats().getTotal(), equalTo(3L + i)); + assertThat(shard.refreshStats().getTotal(), equalTo(2L + i)); assertThat(shard.refreshStats().getTotalTimeInMillis(), greaterThanOrEqualTo(initialTotalTime)); } long refreshCount = shard.refreshStats().getTotal(); From e6f873c620de1b95bc44ee111a581f6ebecd2b45 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 6 Feb 2018 18:00:14 +0100 Subject: [PATCH 07/13] Remove feature parsing for GetIndicesAction (#28535) Removes dead code. Follow-up of #24723 --- .../admin/indices/get/GetIndexRequest.java | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java index 43beba5b670..477656d96cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java @@ -34,9 +34,9 @@ import java.util.List; */ public class GetIndexRequest extends ClusterInfoRequest { public enum Feature { - ALIASES((byte) 0, "_aliases", "_alias"), - MAPPINGS((byte) 1, "_mappings", "_mapping"), - SETTINGS((byte) 2, "_settings"); + ALIASES((byte) 0), + MAPPINGS((byte) 1), + SETTINGS((byte) 2); private static final Feature[] FEATURES = new Feature[Feature.values().length]; @@ -47,52 +47,22 @@ public class GetIndexRequest extends ClusterInfoRequest { } } - private final List validNames; - private final String preferredName; private final byte id; - Feature(byte id, String... validNames) { - assert validNames != null && validNames.length > 0; + Feature(byte id) { this.id = id; - this.validNames = Arrays.asList(validNames); - this.preferredName = validNames[0]; } public byte id() { return id; } - public String preferredName() { - return preferredName; - } - - public boolean validName(String name) { - return this.validNames.contains(name); - } - - public static Feature fromName(String name) { - for (Feature feature : Feature.values()) { - if (feature.validName(name)) { - return feature; - } - } - throw new IllegalArgumentException("No endpoint or operation is available at [" + name + "]"); - } - public static Feature fromId(byte id) { if (id < 0 || id >= FEATURES.length) { throw new IllegalArgumentException("No mapping for id [" + id + "]"); } return FEATURES[id]; } - - public static Feature[] convertToFeatures(String... featureNames) { - Feature[] features = new Feature[featureNames.length]; - for (int i = 0; i < featureNames.length; i++) { - features[i] = Feature.fromName(featureNames[i]); - } - return features; - } } private static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS }; From 6b4ea4e6fb0250f2c8576492c05b517351b18988 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 6 Feb 2018 12:12:48 -0700 Subject: [PATCH 08/13] Add 6.2.1 version constant --- server/src/main/java/org/elasticsearch/Version.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 42f6dea2277..b6681ef003d 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -152,6 +152,8 @@ public class Version implements Comparable { public static final Version V_6_1_4 = new Version(V_6_1_4_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_2_0_ID = 6020099; public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); + public static final int V_6_2_1_ID = 6020199; + public static final Version V_6_2_1 = new Version(V_6_2_1_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_6_3_0_ID = 6030099; public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_7_0_0_alpha1_ID = 7000001; @@ -174,6 +176,8 @@ public class Version implements Comparable { return V_7_0_0_alpha1; case V_6_3_0_ID: return V_6_3_0; + case V_6_2_1_ID: + return V_6_2_1; case V_6_2_0_ID: return V_6_2_0; case V_6_1_4_ID: From 64adaffe1159e6753ec41b6bbc3d66a6ec326aa2 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 6 Feb 2018 15:33:03 -0700 Subject: [PATCH 09/13] [TEST] Expand failure message for wildfly integration tests --- .../src/test/java/org/elasticsearch/wildfly/WildflyIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java index b205a97c40f..3ab992aeec3 100644 --- a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java +++ b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java @@ -26,6 +26,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -78,7 +79,9 @@ public class WildflyIT extends LuceneTestCase { } put.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); try (CloseableHttpResponse response = client.execute(put)) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(201)); + int status = response.getStatusLine().getStatusCode(); + assertThat("expected a 201 response but got: " + status + " - body: " + EntityUtils.toString(response.getEntity()), + status, equalTo(201)); } final HttpGet get = new HttpGet(new URI(str)); From c2fcf15d9de9b3cb20cc57b5d0e889939802f325 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 6 Feb 2018 17:38:26 -0500 Subject: [PATCH 10/13] Fix the ability to remove old plugin We now read the plugin descriptor when removing an old plugin. This is to check if we are removing a plugin that is extended by another plugin. However, when reading the descriptor we enforce that it is of the same version that we are. This is not the case when a user has upgraded Elasticsearch and is now trying to remove an old plugin. This commit fixes this by skipping the version enforcement when reading the plugin descriptor only when removing a plugin. Relates #28540 --- .../plugins/RemovePluginCommand.java | 2 +- .../plugins/RemovePluginCommandTests.java | 40 ++++++++++++++----- .../org/elasticsearch/plugins/PluginInfo.java | 29 ++++++++++---- .../elasticsearch/plugins/PluginsService.java | 26 ++++++++++-- 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java index 4cd83e329b1..ba85173f325 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java @@ -86,7 +86,7 @@ class RemovePluginCommand extends EnvironmentAwareCommand { // first make sure nothing extends this plugin List usedBy = new ArrayList<>(); - Set bundles = PluginsService.getPluginBundles(env.pluginsFile()); + Set bundles = PluginsService.getPluginBundles(env.pluginsFile(), false); for (PluginsService.Bundle bundle : bundles) { for (String extendedPlugin : bundle.plugin.getExtendedPlugins()) { if (extendedPlugin.equals(pluginName)) { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index d15e0e642c8..13506cf986a 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.junit.Before; import java.io.BufferedReader; @@ -41,6 +42,7 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; @LuceneTestCase.SuppressFileSystems("*") @@ -78,19 +80,27 @@ public class RemovePluginCommandTests extends ESTestCase { env = TestEnvironment.newEnvironment(settings); } - void createPlugin(String name) throws Exception { + void createPlugin(String name) throws IOException { createPlugin(env.pluginsFile(), name); } - void createPlugin(Path path, String name) throws Exception { + void createPlugin(String name, Version version) throws IOException { + createPlugin(env.pluginsFile(), name, version); + } + + void createPlugin(Path path, String name) throws IOException { + createPlugin(path, name, Version.CURRENT); + } + + void createPlugin(Path path, String name, Version version) throws IOException { PluginTestUtil.writePluginProperties( - path.resolve(name), - "description", "dummy", - "name", name, - "version", "1.0", - "elasticsearch.version", Version.CURRENT.toString(), - "java.version", System.getProperty("java.specification.version"), - "classname", "SomeClass"); + path.resolve(name), + "description", "dummy", + "name", name, + "version", "1.0", + "elasticsearch.version", version.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "SomeClass"); } void createMetaPlugin(String name, String... plugins) throws Exception { @@ -137,6 +147,18 @@ public class RemovePluginCommandTests extends ESTestCase { assertRemoveCleaned(env); } + public void testRemoveOldVersion() throws Exception { + createPlugin( + "fake", + VersionUtils.randomVersionBetween( + random(), + Version.CURRENT.minimumIndexCompatibilityVersion(), + VersionUtils.getPreviousVersion())); + removePlugin("fake", home, randomBoolean()); + assertThat(Files.exists(env.pluginsFile().resolve("fake")), equalTo(false)); + assertRemoveCleaned(env); + } + public void testBasicMeta() throws Exception { createMetaPlugin("meta", "fake1"); createPlugin("other"); diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 42c9df6d3dd..11cd2514869 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -181,6 +181,19 @@ public class PluginInfo implements Writeable, ToXContentObject { * @throws IOException if an I/O exception occurred reading the plugin descriptor */ public static PluginInfo readFromProperties(final Path path) throws IOException { + return readFromProperties(path, true); + } + + /** + * Reads and validates the plugin descriptor file. If {@code enforceVersion} is false then version enforcement for the plugin descriptor + * is skipped. + * + * @param path the path to the root directory for the plugin + * @param enforceVersion whether or not to enforce the version when reading plugin descriptors + * @return the plugin info + * @throws IOException if an I/O exception occurred reading the plugin descriptor + */ + static PluginInfo readFromProperties(final Path path, final boolean enforceVersion) throws IOException { final Path descriptor = path.resolve(ES_PLUGIN_PROPERTIES); final Map propsMap; @@ -214,7 +227,7 @@ public class PluginInfo implements Writeable, ToXContentObject { "property [elasticsearch.version] is missing for plugin [" + name + "]"); } final Version esVersion = Version.fromString(esVersionString); - if (esVersion.equals(Version.CURRENT) == false) { + if (enforceVersion && esVersion.equals(Version.CURRENT) == false) { final String message = String.format( Locale.ROOT, "plugin [%s] is incompatible with version [%s]; was designed for version [%s]", @@ -258,12 +271,12 @@ public class PluginInfo implements Writeable, ToXContentObject { break; default: final String message = String.format( - Locale.ROOT, - "property [%s] must be [%s], [%s], or unspecified but was [%s]", - "has_native_controller", - "true", - "false", - hasNativeControllerValue); + Locale.ROOT, + "property [%s] must be [%s], [%s], or unspecified but was [%s]", + "has_native_controller", + "true", + "false", + hasNativeControllerValue); throw new IllegalArgumentException(message); } } @@ -277,7 +290,7 @@ public class PluginInfo implements Writeable, ToXContentObject { requiresKeystore = Booleans.parseBoolean(requiresKeystoreValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("property [requires.keystore] must be [true] or [false]," + - " but was [" + requiresKeystoreValue + "]", e); + " but was [" + requiresKeystoreValue + "]", e); } if (propsMap.isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index d60d01273bb..d70b40d701d 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -317,7 +317,27 @@ public class PluginsService extends AbstractComponent { } } - static Set getPluginBundles(Path pluginsDirectory) throws IOException { + /** + * Get the plugin bundles from the specified directory. + * + * @param pluginsDirectory the directory + * @return the set of plugin bundles in the specified directory + * @throws IOException if an I/O exception occurs reading the plugin bundles + */ + static Set getPluginBundles(final Path pluginsDirectory) throws IOException { + return getPluginBundles(pluginsDirectory, true); + } + + /** + * Get the plugin bundles from the specified directory. If {@code enforceVersion} is true, then the version in each plugin descriptor + * must match the current version. + * + * @param pluginsDirectory the directory + * @param enforceVersion whether or not to enforce the version when reading plugin descriptors + * @return the set of plugin bundles in the specified directory + * @throws IOException if an I/O exception occurs reading the plugin bundles + */ + static Set getPluginBundles(final Path pluginsDirectory, final boolean enforceVersion) throws IOException { Logger logger = Loggers.getLogger(PluginsService.class); Set bundles = new LinkedHashSet<>(); @@ -326,10 +346,10 @@ public class PluginsService extends AbstractComponent { logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); final PluginInfo info; try { - info = PluginInfo.readFromProperties(plugin); + info = PluginInfo.readFromProperties(plugin, enforceVersion); } catch (IOException e) { throw new IllegalStateException("Could not load plugin descriptor for existing plugin [" - + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); + + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); } if (bundles.add(new Bundle(info, plugin)) == false) { throw new IllegalStateException("duplicate plugin: " + info); From 305b87b4b74483a4f7a5c24d209f2bef19256291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 7 Feb 2018 09:33:21 +0100 Subject: [PATCH 11/13] Make internal Rounding fields final (#28532) The fields in the internal rounding classes can be made final with very minor adjustments to how they are read from a StreamInput. --- .../common/rounding/Rounding.java | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java b/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java index 808e59ef02a..93be4cc65e2 100644 --- a/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java +++ b/server/src/main/java/org/elasticsearch/common/rounding/Rounding.java @@ -21,7 +21,7 @@ package org.elasticsearch.common.rounding; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; import org.joda.time.DateTimeField; import org.joda.time.DateTimeZone; @@ -33,7 +33,7 @@ import java.util.Objects; /** * A strategy for rounding long values. */ -public abstract class Rounding implements Streamable { +public abstract class Rounding implements Writeable { public abstract byte id(); @@ -107,13 +107,10 @@ public abstract class Rounding implements Streamable { static final byte ID = 1; - private DateTimeUnit unit; - private DateTimeField field; - private DateTimeZone timeZone; - private boolean unitRoundsToMidnight; - - TimeUnitRounding() { // for serialization - } + private final DateTimeUnit unit; + private final DateTimeField field; + private final DateTimeZone timeZone; + private final boolean unitRoundsToMidnight; TimeUnitRounding(DateTimeUnit unit, DateTimeZone timeZone) { this.unit = unit; @@ -122,6 +119,13 @@ public abstract class Rounding implements Streamable { this.timeZone = timeZone; } + TimeUnitRounding(StreamInput in) throws IOException { + unit = DateTimeUnit.resolve(in.readByte()); + timeZone = DateTimeZone.forID(in.readString()); + field = unit.field(timeZone); + unitRoundsToMidnight = field.getDurationField().getUnitMillis() > 60L * 60L * 1000L; + } + @Override public byte id() { return ID; @@ -237,14 +241,6 @@ public abstract class Rounding implements Streamable { return next; } - @Override - public void readFrom(StreamInput in) throws IOException { - unit = DateTimeUnit.resolve(in.readByte()); - timeZone = DateTimeZone.forID(in.readString()); - field = unit.field(timeZone); - unitRoundsToMidnight = field.getDurationField().getUnitMillis() > 60L * 60L * 1000L; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeByte(unit.id()); @@ -278,11 +274,8 @@ public abstract class Rounding implements Streamable { static final byte ID = 2; - private long interval; - private DateTimeZone timeZone; - - TimeIntervalRounding() { // for serialization - } + private final long interval; + private final DateTimeZone timeZone; TimeIntervalRounding(long interval, DateTimeZone timeZone) { if (interval < 1) @@ -291,6 +284,11 @@ public abstract class Rounding implements Streamable { this.timeZone = timeZone; } + TimeIntervalRounding(StreamInput in) throws IOException { + interval = in.readVLong(); + timeZone = DateTimeZone.forID(in.readString()); + } + @Override public byte id() { return ID; @@ -374,12 +372,6 @@ public abstract class Rounding implements Streamable { return timeZone.convertLocalToUTC(next, false); } - @Override - public void readFrom(StreamInput in) throws IOException { - interval = in.readVLong(); - timeZone = DateTimeZone.forID(in.readString()); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(interval); @@ -415,11 +407,10 @@ public abstract class Rounding implements Streamable { Rounding rounding = null; byte id = in.readByte(); switch (id) { - case TimeUnitRounding.ID: rounding = new TimeUnitRounding(); break; - case TimeIntervalRounding.ID: rounding = new TimeIntervalRounding(); break; + case TimeUnitRounding.ID: rounding = new TimeUnitRounding(in); break; + case TimeIntervalRounding.ID: rounding = new TimeIntervalRounding(in); break; default: throw new ElasticsearchException("unknown rounding id [" + id + "]"); } - rounding.readFrom(in); return rounding; } From c0886cf7c6d567e33d2d612fe8b7617c7e3718ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 7 Feb 2018 17:26:08 +0100 Subject: [PATCH 12/13] [Tests] Relax assertion in SuggestStatsIT (#28544) The test expects suggest times in milliseconds that are strictly positive. Internally they are measured in nanos, it is possible that on really fast execution this is rounded to 0L, so this should also be an accepted value. Closes #28543 --- .../org/elasticsearch/index/suggest/stats/SuggestStatsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/server/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index 25eb6df4c27..7b7e7a41783 100644 --- a/server/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/server/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -112,7 +112,7 @@ public class SuggestStatsIT extends ESIntegTestCase { logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime); // check suggest time - assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L)); + assertThat(suggest.getSuggestTimeInMillis(), greaterThanOrEqualTo(0L)); // the upperbound is num shards * total time since we do searches in parallel assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime))); @@ -124,7 +124,7 @@ public class SuggestStatsIT extends ESIntegTestCase { logger.info("evaluating {}", stat.getNode()); if (nodeIdsWithIndex.contains(stat.getNode().getId())) { assertThat(suggestStats.getSuggestCount(), greaterThan(0L)); - assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L)); + assertThat(suggestStats.getSuggestTimeInMillis(), greaterThanOrEqualTo(0L)); num++; } else { assertThat(suggestStats.getSuggestCount(), equalTo(0L)); From 2023c98beadab0e18fddfa5cf429c707b173c883 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 6 Feb 2018 13:47:06 +0100 Subject: [PATCH 13/13] Added more parameter to PersistentTaskPlugin#getPersistentTasksExecutor(...) --- server/src/main/java/org/elasticsearch/node/Node.java | 2 +- .../org/elasticsearch/plugins/PersistentTaskPlugin.java | 5 ++++- .../persistent/TestPersistentTasksPlugin.java | 8 ++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 15c0428d259..fd1a2159b07 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -468,7 +468,7 @@ public class Node implements Closeable { final List> tasksExecutors = pluginsService .filterPlugins(PersistentTaskPlugin.class).stream() - .map(p -> p.getPersistentTasksExecutor(clusterService)) + .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client)) .flatMap(List::stream) .collect(toList()); diff --git a/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java index c402b907ffd..5e3319a2bc5 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java @@ -18,8 +18,10 @@ */ package org.elasticsearch.plugins; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.threadpool.ThreadPool; import java.util.Collections; import java.util.List; @@ -32,7 +34,8 @@ public interface PersistentTaskPlugin { /** * Returns additional persistent tasks executors added by this plugin. */ - default List> getPersistentTasksExecutor(ClusterService clusterService) { + default List> getPersistentTasksExecutor(ClusterService clusterService, + ThreadPool threadPool, Client client) { return Collections.emptyList(); } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index ca3e840028c..35c26105672 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -51,25 +51,20 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -96,7 +91,8 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin, P } @Override - public List> getPersistentTasksExecutor(ClusterService clusterService) { + public List> getPersistentTasksExecutor(ClusterService clusterService, + ThreadPool threadPool, Client client) { return Collections.singletonList(new TestPersistentTasksExecutor(Settings.EMPTY, clusterService)); }