From c1cd964458b31fe5058eecea0ff06644cabec3c1 Mon Sep 17 00:00:00 2001 From: Dimitrios Athanasiou Date: Fri, 23 Dec 2016 02:21:31 +0000 Subject: [PATCH] fixed compile errors due to upstream changes in elasticsearch itself Note that the change in elasticsearch allowed us to store scheduler config's query and scriptFields as typed objects instead of ByteReference. Original commit: elastic/x-pack-elasticsearch@38c5aef2ef1db133b12f4011800907da1084a8ae --- .../prelert/job/metadata/PrelertMetadata.java | 10 +- .../prelert/scheduler/SchedulerConfig.java | 170 +++++++----------- .../http/HttpDataExtractorFactory.java | 7 +- .../scheduler/SchedulerConfigTests.java | 26 ++- .../support/AbstractSerializingTestCase.java | 10 +- .../support/AbstractStreamableTestCase.java | 11 +- .../AbstractStreamableXContentTestCase.java | 10 +- .../AbstractWireSerializingTestCase.java | 17 +- 8 files changed, 119 insertions(+), 142 deletions(-) diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java index 3f4e7eadb7c..bec09ef5d60 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java @@ -24,11 +24,11 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.xpack.prelert.job.Job; import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.scheduler.ScheduledJobValidator; import org.elasticsearch.xpack.prelert.scheduler.Scheduler; import org.elasticsearch.xpack.prelert.scheduler.SchedulerConfig; import org.elasticsearch.xpack.prelert.scheduler.SchedulerStatus; -import org.elasticsearch.xpack.prelert.job.messages.Messages; -import org.elasticsearch.xpack.prelert.scheduler.ScheduledJobValidator; import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; import java.io.IOException; @@ -289,10 +289,8 @@ public class PrelertMetadata implements MetaData.Custom { } ScheduledJobValidator.validate(schedulerConfig, job); - // Check the query, aggregations and script_fields can be built - schedulerConfig.buildQuery(searchRequestParsers.queryParsers); - schedulerConfig.buildAggregations(searchRequestParsers.queryParsers, searchRequestParsers.aggParsers); - schedulerConfig.buildScriptFields(searchRequestParsers.queryParsers); + // Check that aggregations can be built + schedulerConfig.buildAggregations(searchRequestParsers.aggParsers); return putScheduler(new Scheduler(schedulerConfig, SchedulerStatus.STOPPED)); } diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfig.java index 8f0d7483694..a56be204551 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfig.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfig.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -36,6 +35,7 @@ import org.elasticsearch.xpack.prelert.utils.PrelertStrings; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -81,11 +81,8 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { PARSER.declareStringArray(Builder::setTypes, TYPES); PARSER.declareLong(Builder::setQueryDelay, QUERY_DELAY); PARSER.declareLong(Builder::setFrequency, FREQUENCY); - PARSER.declareField((parser, builder, aVoid) -> { - XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); - XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); - builder.setQuery(contentBuilder.bytes()); - }, QUERY, ObjectParser.ValueType.OBJECT); + PARSER.declareObject(Builder::setQuery, + (p, c) -> new QueryParseContext(p, ParseFieldMatcher.STRICT).parseInnerQueryBuilder().get(), QUERY); PARSER.declareField((parser, builder, aVoid) -> { XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); @@ -96,11 +93,14 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); builder.setAggregations(contentBuilder.bytes()); }, AGGS, ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, builder, aVoid) -> { - XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); - XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser); - builder.setScriptFields(contentBuilder.bytes()); - }, SCRIPT_FIELDS, ObjectParser.ValueType.OBJECT); + PARSER.declareObject(Builder::setScriptFields, (p, c) -> { + List parsedScriptFields = new ArrayList<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + parsedScriptFields.add(new SearchSourceBuilder.ScriptField(new QueryParseContext(p, ParseFieldMatcher.STRICT))); + } + Collections.sort(parsedScriptFields, Comparator.comparing(f -> f.fieldName())); + return parsedScriptFields; + }, SCRIPT_FIELDS); PARSER.declareInt(Builder::setScrollSize, SCROLL_SIZE); } @@ -119,13 +119,14 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { private final List indexes; private final List types; - private final BytesReference query; + private final QueryBuilder query; private final BytesReference aggregations; - private final BytesReference scriptFields; + private final List scriptFields; private final Integer scrollSize; private SchedulerConfig(String id, String jobId, Long queryDelay, Long frequency, List indexes, List types, - BytesReference query, BytesReference aggregations, BytesReference scriptFields, Integer scrollSize) { + QueryBuilder query, BytesReference aggregations, List scriptFields, + Integer scrollSize) { this.id = id; this.jobId = jobId; this.queryDelay = queryDelay; @@ -153,9 +154,13 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { } else { this.types = null; } - this.query = in.readOptionalBytesReference(); + this.query = in.readNamedWriteable(QueryBuilder.class); this.aggregations = in.readOptionalBytesReference(); - this.scriptFields = in.readOptionalBytesReference(); + if (in.readBoolean()) { + this.scriptFields = in.readList(SearchSourceBuilder.ScriptField::new); + } else { + this.scriptFields = null; + } this.scrollSize = in.readOptionalVInt(); } @@ -199,41 +204,24 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { return this.scrollSize; } - Map getQueryAsMap() { - return XContentHelper.convertToMap(query, true).v2(); - } - Map getAggregationsAsMap() { return XContentHelper.convertToMap(aggregations, true).v2(); } - Map getScriptFieldsAsMap() { - return XContentHelper.convertToMap(scriptFields, true).v2(); - } - - public QueryBuilder buildQuery(IndicesQueriesRegistry queryParsers) { - if (query == null) { - return QueryBuilders.matchAllQuery(); - } - XContentParser parser = createParser(QUERY, query); - QueryParseContext queryParseContext = new QueryParseContext(queryParsers, parser, ParseFieldMatcher.STRICT); - try { - return queryParseContext.parseInnerQueryBuilder().orElse(QueryBuilders.matchAllQuery()); - } catch (IOException e) { - throw ExceptionsHelper.parseException(QUERY, e); - } + public QueryBuilder getQuery() { + return query; } public boolean hasAggregations() { return aggregations != null; } - public AggregatorFactories.Builder buildAggregations(IndicesQueriesRegistry queryParsers, AggregatorParsers aggParsers) { + public AggregatorFactories.Builder buildAggregations(AggregatorParsers aggParsers) { if (!hasAggregations()) { return null; } XContentParser parser = createParser(AGGREGATIONS, aggregations); - QueryParseContext queryParseContext = new QueryParseContext(queryParsers, parser, ParseFieldMatcher.STRICT); + QueryParseContext queryParseContext = new QueryParseContext(parser, ParseFieldMatcher.STRICT); try { return aggParsers.parseAggregators(queryParseContext); } catch (IOException e) { @@ -241,26 +229,13 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { } } - public List buildScriptFields(IndicesQueriesRegistry queryParsers) { - if (scriptFields == null) { - return Collections.emptyList(); - } - List parsedScriptFields = new ArrayList<>(); - XContentParser parser = createParser(SCRIPT_FIELDS, scriptFields); - try { - QueryParseContext queryParseContext = new QueryParseContext(queryParsers, parser, ParseFieldMatcher.STRICT); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - parsedScriptFields.add(new SearchSourceBuilder.ScriptField(queryParseContext)); - } - } catch (IOException e) { - throw ExceptionsHelper.parseException(SCRIPT_FIELDS, e); - } - return parsedScriptFields; + public List getScriptFields() { + return scriptFields == null ? Collections.emptyList() : scriptFields; } private XContentParser createParser(ParseField parseField, BytesReference bytesReference) { try { - return XContentFactory.xContent(query).createParser(NamedXContentRegistry.EMPTY, query); + return XContentFactory.xContent(bytesReference).createParser(NamedXContentRegistry.EMPTY, bytesReference); } catch (IOException e) { throw ExceptionsHelper.parseException(parseField, e); } @@ -284,9 +259,14 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { } else { out.writeBoolean(false); } - out.writeOptionalBytesReference(query); + out.writeNamedWriteable(query); out.writeOptionalBytesReference(aggregations); - out.writeOptionalBytesReference(scriptFields); + if (scriptFields != null) { + out.writeBoolean(true); + out.writeList(scriptFields); + } else { + out.writeBoolean(false); + } out.writeOptionalVInt(scrollSize); } @@ -301,30 +281,24 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field(ID.getPreferredName(), id); builder.field(Job.ID.getPreferredName(), jobId); - if (queryDelay != null) { - builder.field(QUERY_DELAY.getPreferredName(), queryDelay); - } + builder.field(QUERY_DELAY.getPreferredName(), queryDelay); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency); } - if (indexes != null) { - builder.field(INDEXES.getPreferredName(), indexes); - } - if (types != null) { - builder.field(TYPES.getPreferredName(), types); - } - if (query != null) { - builder.field(QUERY.getPreferredName(), getQueryAsMap()); - } + builder.field(INDEXES.getPreferredName(), indexes); + builder.field(TYPES.getPreferredName(), types); + builder.field(QUERY.getPreferredName(), query); if (aggregations != null) { builder.field(AGGREGATIONS.getPreferredName(), getAggregationsAsMap()); } if (scriptFields != null) { - builder.field(SCRIPT_FIELDS.getPreferredName(), getScriptFieldsAsMap()); - } - if (scrollSize != null) { - builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); + builder.startObject(SCRIPT_FIELDS.getPreferredName()); + for (SearchSourceBuilder.ScriptField scriptField : scriptFields) { + scriptField.toXContent(builder, params); + } + builder.endObject(); } + builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); return builder; } @@ -369,18 +343,16 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { private String id; private String jobId; - private Long queryDelay; + private Long queryDelay = DEFAULT_ELASTICSEARCH_QUERY_DELAY; private Long frequency; private List indexes = Collections.emptyList(); private List types = Collections.emptyList(); - private BytesReference query; + private QueryBuilder query = QueryBuilders.matchAllQuery(); private BytesReference aggregations; - private BytesReference scriptFields; - private Integer scrollSize; + private List scriptFields; + private Integer scrollSize = DEFAULT_SCROLL_SIZE; public Builder() { - setQueryDelay(DEFAULT_ELASTICSEARCH_QUERY_DELAY); - setScrollSize(DEFAULT_SCROLL_SIZE); } public Builder(String id, String jobId) { @@ -436,12 +408,16 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { this.frequency = frequency; } - private void setQuery(BytesReference query) { - this.query = Objects.requireNonNull(query); + public void setQuery(QueryBuilder query) { + this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName()); } - public void setQuery(QueryBuilder query) { - this.query = xContentToBytes(query); + private void setAggregations(BytesReference aggregations) { + this.aggregations = Objects.requireNonNull(aggregations); + } + + public void setAggregations(AggregatorFactories.Builder aggregations) { + this.aggregations = xContentToBytes(aggregations); } private BytesReference xContentToBytes(ToXContent value) { @@ -453,30 +429,10 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { } } - private void setAggregations(BytesReference aggregations) { - this.aggregations = Objects.requireNonNull(aggregations); - } - - public void setAggregations(AggregatorFactories.Builder aggregations) { - this.aggregations = xContentToBytes(aggregations); - } - - private void setScriptFields(BytesReference scriptFields) { - this.scriptFields = Objects.requireNonNull(scriptFields); - } - public void setScriptFields(List scriptFields) { - try { - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - jsonBuilder.startObject(); - for (SearchSourceBuilder.ScriptField scriptField : scriptFields) { - scriptField.toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS); - } - jsonBuilder.endObject(); - this.scriptFields = jsonBuilder.bytes(); - } catch (IOException e) { - throw new RuntimeException(e); - } + List sorted = new ArrayList<>(scriptFields); + Collections.sort(sorted, Comparator.comparing(f -> f.fieldName())); + this.scriptFields = sorted; } public void setScrollSize(int scrollSize) { @@ -507,12 +463,6 @@ public class SchedulerConfig extends ToXContentToBytes implements Writeable { String msg = Messages.getMessage(Messages.SCHEDULER_CONFIG_INVALID_OPTION_VALUE, fieldName, value); throw new IllegalArgumentException(msg); } - - private static ElasticsearchException notSupportedValue(ParseField field, String key) { - String msg = Messages.getMessage(key, field.getPreferredName()); - throw new IllegalArgumentException(msg); - } - } } diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/http/HttpDataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/http/HttpDataExtractorFactory.java index b5bc2aba1a7..6e939e1cc10 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/http/HttpDataExtractorFactory.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/scheduler/http/HttpDataExtractorFactory.java @@ -44,10 +44,9 @@ public class HttpDataExtractorFactory implements DataExtractorFactory { public DataExtractor newExtractor(SchedulerConfig schedulerConfig, Job job) { String timeField = job.getDataDescription().getTimeField(); ElasticsearchQueryBuilder queryBuilder = new ElasticsearchQueryBuilder( - xContentToJson(schedulerConfig.buildQuery(searchRequestParsers.queryParsers)), - stringifyAggregations(schedulerConfig.buildAggregations(searchRequestParsers.queryParsers, - searchRequestParsers.aggParsers)), - stringifyScriptFields(schedulerConfig.buildScriptFields(searchRequestParsers.queryParsers)), + xContentToJson(schedulerConfig.getQuery()), + stringifyAggregations(schedulerConfig.buildAggregations(searchRequestParsers.aggParsers)), + stringifyScriptFields(schedulerConfig.getScriptFields()), timeField); HttpRequester httpRequester = new HttpRequester(); ElasticsearchUrlBuilder urlBuilder = ElasticsearchUrlBuilder diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfigTests.java index 0c86bf31080..9664e53b1ee 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfigTests.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/scheduler/SchedulerConfigTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.prelert.scheduler; import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -39,6 +38,16 @@ public class SchedulerConfigTests extends AbstractSerializingTestCase scriptFields = new ArrayList<>(scriptsSize); + for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) { + scriptFields.add(new SearchSourceBuilder.ScriptField(randomAsciiOfLength(10), new Script(randomAsciiOfLength(10)), + randomBoolean())); + } + builder.setScriptFields(scriptFields); if (randomBoolean()) { builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE)); } @@ -70,19 +79,10 @@ public class SchedulerConfigTests extends AbstractSerializingTestCase matcher).build(); } - public void testToXContent_GivenQueryAggsAndScriptFields() throws IOException { + public void testToXContent_GivenAggregations() throws IOException { SchedulerConfig.Builder builder = new SchedulerConfig.Builder(randomValidSchedulerId(), randomAsciiOfLength(10)); builder.setIndexes(randomStringList(1, 10)); builder.setTypes(randomStringList(1, 10)); - builder.setQuery(QueryBuilders.matchAllQuery()); - - int scriptsSize = randomInt(3); - List scriptFields = new ArrayList<>(scriptsSize); - for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) { - scriptFields.add(new SearchSourceBuilder.ScriptField(randomAsciiOfLength(10), new Script(randomAsciiOfLength(10)), - randomBoolean())); - } - builder.setScriptFields(scriptFields); AggregatorFactories.Builder aggsBuilder = new AggregatorFactories.Builder(); aggsBuilder.addAggregator(AggregationBuilders.avg(randomAsciiOfLength(10))); @@ -94,12 +94,10 @@ public class SchedulerConfigTests extends AbstractSerializingTestCase extends AbstractWireSerializingTestCase { + protected static final NamedXContentRegistry NAMED_X_CONTENT_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + NAMED_X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); + } + /** * Generic test that creates new instance from the test instance and checks * both for equality and asserts equality on the two queries. @@ -42,7 +50,7 @@ public abstract class AbstractSerializingTestCase extends ESTestCase { protected static final int NUMBER_OF_TESTQUERIES = 20; + private static final NamedWriteableRegistry NAMED_WRITEABLE_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + NAMED_WRITEABLE_REGISTRY = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + protected abstract T createTestInstance(); protected abstract T createBlankInstance(); @@ -77,8 +85,7 @@ public abstract class AbstractStreamableTestCase extends E private T copyInstance(T instance) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { instance.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), - new NamedWriteableRegistry(Collections.emptyList()))) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), NAMED_WRITEABLE_REGISTRY)) { T newInstance = createBlankInstance(); newInstance.readFrom(in); return newInstance; diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableXContentTestCase.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableXContentTestCase.java index 4dda6de23b2..8ba4113d05a 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableXContentTestCase.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableXContentTestCase.java @@ -10,12 +10,14 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchModule; import java.io.IOException; import java.util.Collections; @@ -23,6 +25,12 @@ import java.util.Map; public abstract class AbstractStreamableXContentTestCase extends AbstractStreamableTestCase { + protected static final NamedXContentRegistry NAMED_X_CONTENT_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + NAMED_X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); + } + /** * Generic test that creates new instance from the test instance and checks * both for equality and asserts equality on the two queries. @@ -42,7 +50,7 @@ public abstract class AbstractStreamableXContentTestCase extends ESTestCase { protected static final int NUMBER_OF_TESTQUERIES = 20; + private static final NamedWriteableRegistry NAMED_WRITEABLE_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + NAMED_WRITEABLE_REGISTRY = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + protected abstract T createTestInstance(); protected abstract Reader instanceReader(); @@ -77,8 +87,7 @@ public abstract class AbstractWireSerializingTestCase