diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java index 8465ae83428..34bcb595c20 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java @@ -19,16 +19,20 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig; +import org.elasticsearch.client.dataframe.transforms.util.TimeUtil; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.time.Instant; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -40,6 +44,8 @@ public class DataFrameTransformConfig implements ToXContentObject { public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DEST = new ParseField("dest"); public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField VERSION = new ParseField("version"); + public static final ParseField CREATE_TIME = new ParseField("create_time"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); @@ -48,6 +54,8 @@ public class DataFrameTransformConfig implements ToXContentObject { private final DestConfig dest; private final PivotConfig pivotConfig; private final String description; + private final Version transformVersion; + private final Instant createTime; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_frame_transform", true, @@ -57,7 +65,9 @@ public class DataFrameTransformConfig implements ToXContentObject { DestConfig dest = (DestConfig) args[2]; PivotConfig pivotConfig = (PivotConfig) args[3]; String description = (String)args[4]; - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + Instant createTime = (Instant)args[5]; + String transformVersion = (String)args[6]; + return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, createTime, transformVersion); }); static { @@ -66,6 +76,9 @@ public class DataFrameTransformConfig implements ToXContentObject { PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST); PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); + PARSER.declareField(optionalConstructorArg(), + p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + PARSER.declareString(optionalConstructorArg(), VERSION); } public static DataFrameTransformConfig fromXContent(final XContentParser parser) { @@ -84,19 +97,23 @@ public class DataFrameTransformConfig implements ToXContentObject { * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new DataFrameTransformConfig(null, source, null, pivotConfig, null); + return new DataFrameTransformConfig(null, source, null, pivotConfig, null, null, null); } DataFrameTransformConfig(final String id, final SourceConfig source, final DestConfig dest, final PivotConfig pivotConfig, - final String description) { + final String description, + final Instant createTime, + final String version) { this.id = id; this.source = source; this.dest = dest; this.pivotConfig = pivotConfig; this.description = description; + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); + this.transformVersion = version == null ? null : Version.fromString(version); } public String getId() { @@ -115,6 +132,14 @@ public class DataFrameTransformConfig implements ToXContentObject { return pivotConfig; } + public Version getVersion() { + return transformVersion; + } + + public Instant getCreateTime() { + return createTime; + } + @Nullable public String getDescription() { return description; @@ -138,6 +163,12 @@ public class DataFrameTransformConfig implements ToXContentObject { if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (transformVersion != null) { + builder.field(VERSION.getPreferredName(), transformVersion); + } builder.endObject(); return builder; } @@ -155,15 +186,17 @@ public class DataFrameTransformConfig implements ToXContentObject { final DataFrameTransformConfig that = (DataFrameTransformConfig) other; return Objects.equals(this.id, that.id) - && Objects.equals(this.source, that.source) - && Objects.equals(this.dest, that.dest) - && Objects.equals(this.description, that.description) - && Objects.equals(this.pivotConfig, that.pivotConfig); + && Objects.equals(this.source, that.source) + && Objects.equals(this.dest, that.dest) + && Objects.equals(this.description, that.description) + && Objects.equals(this.transformVersion, that.transformVersion) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.pivotConfig, that.pivotConfig); } @Override public int hashCode() { - return Objects.hash(id, source, dest, pivotConfig, description); + return Objects.hash(id, source, dest, pivotConfig, description, createTime, transformVersion); } @Override @@ -209,7 +242,7 @@ public class DataFrameTransformConfig implements ToXContentObject { } public DataFrameTransformConfig build() { - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, null, null); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java new file mode 100644 index 00000000000..2470c3f7a4a --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.dataframe.transforms.util; + +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +public final class TimeUtil { + + /** + * Parse out a Date object given the current parser and field name. + * + * @param parser current XContentParser + * @param fieldName the field's preferred name (utilized in exception) + * @return parsed Date object + * @throws IOException from XContentParser + */ + public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return new Date(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + + public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return Instant.ofEpochMilli(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant(); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index 8489d14e101..44af764cc68 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -195,7 +195,7 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase { client::getDataFrameTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); - assertEquals(transform, getResponse.getTransformConfigurations().get(0)); + assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId()); } public void testGetAllAndPageTransforms() throws IOException { @@ -219,7 +219,7 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase { client::getDataFrameTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); - assertEquals(transform, getResponse.getTransformConfigurations().get(1)); + assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId()); getRequest.setPageParams(new PageParams(0,1)); getResponse = execute(getRequest, client::getDataFrameTransform, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java index 1b5228d9622..84782a8a970 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -27,6 +28,7 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.function.Predicate; @@ -36,8 +38,13 @@ import static org.elasticsearch.client.dataframe.transforms.SourceConfigTests.ra public class DataFrameTransformConfigTests extends AbstractXContentTestCase { public static DataFrameTransformConfig randomDataFrameTransformConfig() { - return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), - randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), + randomSourceConfig(), + randomDestConfig(), + PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), + randomBoolean() ? null : Instant.now(), + randomBoolean() ? null : Version.CURRENT.toString()); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 6604e97ed5b..d9ebccfb91f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -478,7 +478,6 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest RestHighLevelClient client = highLevelClient(); - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); @@ -564,7 +563,6 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest public void testGetDataFrameTransform() throws IOException, InterruptedException { createIndex("source-data"); - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java index ee35fe3d21e..19d4d6ab6ee 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -14,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -21,8 +23,10 @@ import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig; import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -42,6 +46,8 @@ public class DataFrameTransformConfig extends AbstractDiffable STRICT_PARSER = createParser(false); private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); private static final int MAX_DESCRIPTION_LENGTH = 1_000; @@ -53,9 +59,17 @@ public class DataFrameTransformConfig extends AbstractDiffable headers; + private Version transformVersion; + private Instant createTime; private final PivotConfig pivotConfig; + private static void validateStrictParsingParams(Object arg, String parameterName) { + if (arg != null) { + throw new IllegalArgumentException("Found [" + parameterName + "], not allowed for strict parsing"); + } + } + private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, (args, optionalId) -> { @@ -74,9 +88,11 @@ public class DataFrameTransformConfig extends AbstractDiffable p.mapStrings(), HEADERS); parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareString(optionalConstructorArg(), DESCRIPTION); - + parser.declareField(optionalConstructorArg(), + p -> TimeUtils.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + parser.declareString(optionalConstructorArg(), VERSION); return parser; } @@ -103,12 +128,14 @@ public class DataFrameTransformConfig extends AbstractDiffable headers, - final PivotConfig pivotConfig, - final String description) { + DataFrameTransformConfig(final String id, + final SourceConfig source, + final DestConfig dest, + final Map headers, + final PivotConfig pivotConfig, + final String description, + final Instant createTime, + final String version){ this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName()); this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName()); this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName()); @@ -123,6 +150,17 @@ public class DataFrameTransformConfig extends AbstractDiffable MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); + this.transformVersion = version == null ? null : Version.fromString(version); + } + + public DataFrameTransformConfig(final String id, + final SourceConfig source, + final DestConfig dest, + final Map headers, + final PivotConfig pivotConfig, + final String description) { + this(id, source, dest, headers, pivotConfig, description, null, null); } public DataFrameTransformConfig(final StreamInput in) throws IOException { @@ -132,6 +170,13 @@ public class DataFrameTransformConfig extends AbstractDiffable headers) { + public DataFrameTransformConfig setHeaders(Map headers) { this.headers = headers; + return this; + } + + public Version getVersion() { + return transformVersion; + } + + public DataFrameTransformConfig setVersion(Version transformVersion) { + this.transformVersion = transformVersion; + return this; + } + + public Instant getCreateTime() { + return createTime; + } + + public DataFrameTransformConfig setCreateTime(Instant createTime) { + ExceptionsHelper.requireNonNull(createTime, CREATE_TIME.getPreferredName()); + this.createTime = Instant.ofEpochMilli(createTime.toEpochMilli()); + return this; } public PivotConfig getPivotConfig() { @@ -179,6 +244,15 @@ public class DataFrameTransformConfig extends AbstractDiffable createDataFrameTransformConfigFromString(pivotTransform, "test_header_injection")); } + public void testPreventCreateTimeInjection() throws IOException { + String pivotTransform = "{" + + " \"create_time\" : " + Instant.now().toEpochMilli() + " }," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"id\": {" + + " \"terms\": {" + + " \"field\": \"id\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } } } }"; + + expectThrows(IllegalArgumentException.class, + () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection")); + } + + public void testPreventVersionInjection() throws IOException { + String pivotTransform = "{" + + " \"version\" : \"7.3.0\"," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"id\": {" + + " \"terms\": {" + + " \"field\": \"id\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } } } }"; + + expectThrows(IllegalArgumentException.class, + () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection")); + } + public void testXContentForInternalStorage() throws IOException { DataFrameTransformConfig dataFrameTransformConfig = randomDataFrameTransformConfig(); diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 805252b465b..103ea2e9100 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -15,6 +15,8 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.core.AcknowledgedResponse; import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformResponse; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsRequest; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse; import org.elasticsearch.client.dataframe.PutDataFrameTransformRequest; @@ -57,6 +59,7 @@ import java.time.ZoneId; import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -118,6 +121,11 @@ abstract class DataFrameIntegTestCase extends ESRestTestCase { return restClient.dataFrame().getDataFrameTransformStats(new GetDataFrameTransformStatsRequest(id), RequestOptions.DEFAULT); } + protected GetDataFrameTransformResponse getDataFrameTransform(String id) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + return restClient.dataFrame().getDataFrameTransform(new GetDataFrameTransformRequest(id), RequestOptions.DEFAULT); + } + protected void waitUntilCheckpoint(String id, long checkpoint) throws Exception { waitUntilCheckpoint(id, checkpoint, TimeValue.timeValueSeconds(30)); } @@ -321,9 +329,11 @@ abstract class DataFrameIntegTestCase extends ESRestTestCase { .build(); } - private class TestRestHighLevelClient extends RestHighLevelClient { + private static class TestRestHighLevelClient extends RestHighLevelClient { + private static final List X_CONTENT_ENTRIES = + new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedXContents(); TestRestHighLevelClient() { - super(client(), restClient -> {}, Collections.emptyList()); + super(client(), restClient -> {}, X_CONTENT_ENTRIES); } } } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index c4c5ca3c130..174a956eb3c 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.dataframe.integration; +import org.elasticsearch.Version; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.core.IndexerState; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; @@ -17,6 +18,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInter import org.junit.After; import java.io.IOException; +import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -58,6 +60,11 @@ public class DataFrameTransformIT extends DataFrameIntegTestCase { assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getIndexerState(), equalTo(IndexerState.STOPPED))); stopDataFrameTransform(config.getId()); + + DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); + assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT)); + Instant now = Instant.now(); + assertTrue("[create_time] is not before current time", storedConfig.getCreateTime().isBefore(now)); deleteDataFrameTransform(config.getId()); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java index 049b0804f45..36023c0f737 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.dataframe.action; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.IndicesOptions; @@ -51,6 +52,7 @@ import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigMa import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -110,8 +112,10 @@ public class TransportPutDataFrameTransformAction .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - DataFrameTransformConfig config = request.getConfig(); - config.setHeaders(filteredHeaders); + DataFrameTransformConfig config = request.getConfig() + .setHeaders(filteredHeaders) + .setCreateTime(Instant.now()) + .setVersion(Version.CURRENT); String transformId = config.getId(); // quick check whether a transform has already been created under that name diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml index 98bd0959179..a017da63312 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -90,6 +90,8 @@ setup: - match: { transforms.0.source.index.0: "airline-data" } - match: { transforms.0.dest.index: "airline-data-by-airline" } - is_true: transforms.0.source.query.match_all + - is_true: transforms.0.create_time + - is_true: transforms.0.version - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" } - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" } - match: { transforms.0.description: "yaml test transform on airline-data" }