diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java index 5d5f67dd65e..40e87b5768b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java @@ -29,23 +29,32 @@ import java.util.Objects; public class PreviewDataFrameTransformResponse { private static final String PREVIEW = "preview"; + private static final String MAPPINGS = "mappings"; @SuppressWarnings("unchecked") public static PreviewDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException { - Object previewDocs = parser.map().get(PREVIEW); - return new PreviewDataFrameTransformResponse((List>) previewDocs); + Map previewMap = parser.mapOrdered(); + Object previewDocs = previewMap.get(PREVIEW); + Object mappings = previewMap.get(MAPPINGS); + return new PreviewDataFrameTransformResponse((List>) previewDocs, (Map) mappings); } private List> docs; + private Map mappings; - public PreviewDataFrameTransformResponse(List> docs) { + public PreviewDataFrameTransformResponse(List> docs, Map mappings) { this.docs = docs; + this.mappings = mappings; } public List> getDocs() { return docs; } + public Map getMappings() { + return mappings; + } + @Override public boolean equals(Object obj) { if (obj == this) { @@ -57,12 +66,12 @@ public class PreviewDataFrameTransformResponse { } PreviewDataFrameTransformResponse other = (PreviewDataFrameTransformResponse) obj; - return Objects.equals(other.docs, docs); + return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings); } @Override public int hashCode() { - return Objects.hashCode(docs); + return Objects.hash(docs, mappings); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index 5ec2265d045..71b6cfe3337 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -60,6 +60,7 @@ import org.junit.After; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; @@ -277,6 +279,7 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase { assertThat(taskState, is(DataFrameTransformTaskState.STOPPED)); } + @SuppressWarnings("unchecked") public void testPreview() throws IOException { String sourceIndex = "transform-source"; createIndex(sourceIndex); @@ -298,6 +301,12 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase { Optional> michel = docs.stream().filter(doc -> "michel".equals(doc.get("reviewer"))).findFirst(); assertTrue(michel.isPresent()); assertEquals(3.6d, (double) michel.get().get("avg_rating"), 0.1d); + + Map mappings = preview.getMappings(); + assertThat(mappings, hasKey("properties")); + Map fields = (Map)mappings.get("properties"); + assertThat(fields.get("reviewer"), equalTo(Collections.singletonMap("type", "keyword"))); + assertThat(fields.get("avg_rating"), equalTo(Collections.singletonMap("type", "double"))); } private DataFrameTransformConfig validDataFrameTransformConfig(String id, String source, String destination) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java index 31f1a26d6f1..28b7e52aac1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,8 +54,13 @@ public class PreviewDataFrameTransformResponseTests extends ESTestCase { } docs.add(doc); } + int numMappingEntries = randomIntBetween(5, 10); + Map mappings = new HashMap<>(numMappingEntries); + for (int i = 0; i < numMappingEntries; i++) { + mappings.put(randomAlphaOfLength(10), Collections.singletonMap("type", randomAlphaOfLength(10))); + } - return new PreviewDataFrameTransformResponse(docs); + return new PreviewDataFrameTransformResponse(docs, mappings); } private void toXContent(PreviewDataFrameTransformResponse response, XContentBuilder builder) throws IOException { @@ -64,6 +70,7 @@ public class PreviewDataFrameTransformResponseTests extends ESTestCase { builder.map(doc); } builder.endArray(); + builder.field("mappings", response.getMappings()); builder.endObject(); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 1301a952676..213bb7a02e2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -447,6 +447,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest // end::preview-data-frame-transform-execute assertNotNull(response.getDocs()); + assertNotNull(response.getMappings()); } { // tag::preview-data-frame-transform-execute-listener diff --git a/docs/reference/data-frames/apis/preview-transform.asciidoc b/docs/reference/data-frames/apis/preview-transform.asciidoc index 5dfe1f2f1d7..a4338d0ef60 100644 --- a/docs/reference/data-frames/apis/preview-transform.asciidoc +++ b/docs/reference/data-frames/apis/preview-transform.asciidoc @@ -90,7 +90,17 @@ The data that is returned for this example is as follows: "customer_id" : "12" } ... - ] + ], + "mappings": { + "properties": { + "max_price": { + "type": "double" + }, + "customer_id": { + "type": "keyword" + } + } + } } ---- // NOTCONSOLE diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java index bb71019eaef..aba75c6a877 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.dataframe.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; @@ -28,6 +29,7 @@ import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -137,11 +139,14 @@ public class PreviewDataFrameTransformAction extends ActionType> docs; + private Map mappings; public static ParseField PREVIEW = new ParseField("preview"); + public static ParseField MAPPINGS = new ParseField("mappings"); static ObjectParser PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new); static { PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW); + PARSER.declareObject(Response::setMappings, (p, c) -> p.mapOrdered(), MAPPINGS); } public Response() {} @@ -151,6 +156,10 @@ public class PreviewDataFrameTransformAction extends ActionType objectMap = in.readMap(); + this.mappings = objectMap == null ? null : Collections.unmodifiableMap(objectMap); + } } public Response(List> docs) { @@ -161,18 +170,56 @@ public class PreviewDataFrameTransformAction extends ActionType(docs); } + public void setMappings(Map mappings) { + this.mappings = Collections.unmodifiableMap(mappings); + } + + /** + * This takes the a {@code Map} of the type "fieldname: fieldtype" and transforms it into the + * typical mapping format. + * + * Example: + * + * input: + * {"field1.subField1": "long", "field2": "keyword"} + * + * output: + * { + * "properties": { + * "field1.subField1": { + * "type": "long" + * }, + * "field2": { + * "type": "keyword" + * } + * } + * } + * @param mappings A Map of the form {"fieldName": "fieldType"} + */ + public void setMappingsFromStringMap(Map mappings) { + Map fieldMappings = new HashMap<>(); + mappings.forEach((k, v) -> fieldMappings.put(k, Collections.singletonMap("type", v))); + this.mappings = Collections.singletonMap("properties", fieldMappings); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeInt(docs.size()); for (Map doc : docs) { out.writeMapWithConsistentOrder(doc); } + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeMap(mappings); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(PREVIEW.getPreferredName(), docs); + if (mappings != null) { + builder.field(MAPPINGS.getPreferredName(), mappings); + } builder.endObject(); return builder; } @@ -188,12 +235,12 @@ public class PreviewDataFrameTransformAction extends ActionType> data = new ArrayList<>(size); for (int i = 0; i < size; i++) { - Map datum = new HashMap<>(); - Map entry = new HashMap<>(); - entry.put("value1", randomIntBetween(1, 100)); - datum.put(randomAlphaOfLength(10), entry); - data.add(datum); + data.add(Collections.singletonMap(randomAlphaOfLength(10), Collections.singletonMap("value1", randomIntBetween(1, 100)))); } - return new Response(data); + + Response response = new Response(data); + if (randomBoolean()) { + size = randomIntBetween(0, 10); + if (randomBoolean()) { + Map mappings = new HashMap<>(size); + for (int i = 0; i < size; i++) { + mappings.put(randomAlphaOfLength(10), Collections.singletonMap("type", randomAlphaOfLength(10))); + } + response.setMappings(mappings); + } else { + Map mappings = new HashMap<>(size); + for (int i = 0; i < size; i++) { + mappings.put(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } + response.setMappingsFromStringMap(mappings); + } + } + + return response; } @Override diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java index e4e5c408e80..0d6097fab6b 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java @@ -105,17 +105,7 @@ public class TransportPreviewDataFrameTransformAction extends Pivot pivot = new Pivot(config.getPivotConfig()); - getPreview(pivot, - config.getSource(), - config.getDestination().getPipeline(), - config.getDestination().getIndex(), - ActionListener.wrap( - previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)), - error -> { - logger.error("Failure gathering preview", error); - listener.onFailure(error); - } - )); + getPreview(pivot, config.getSource(), config.getDestination().getPipeline(), config.getDestination().getIndex(), listener); } @SuppressWarnings("unchecked") @@ -123,7 +113,8 @@ public class TransportPreviewDataFrameTransformAction extends SourceConfig source, String pipeline, String dest, - ActionListener>> listener) { + ActionListener listener) { + final PreviewDataFrameTransformAction.Response previewResponse = new PreviewDataFrameTransformAction.Response(); ActionListener pipelineResponseActionListener = ActionListener.wrap( simulatePipelineResponse -> { List> response = new ArrayList<>(simulatePipelineResponse.getResults().size()); @@ -136,12 +127,14 @@ public class TransportPreviewDataFrameTransformAction extends response.add((Map)XContentMapValues.extractValue("doc._source", tempMap)); } } - listener.onResponse(response); + previewResponse.setDocs(response); + listener.onResponse(previewResponse); }, listener::onFailure ); pivot.deduceMappings(client, source, ActionListener.wrap( deducedMappings -> { + previewResponse.setMappingsFromStringMap(deducedMappings); ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(), ClientHelper.DATA_FRAME_ORIGIN, client, @@ -158,7 +151,8 @@ public class TransportPreviewDataFrameTransformAction extends List> results = pivot.extractResults(agg, deducedMappings, stats) .peek(doc -> doc.keySet().removeIf(k -> k.startsWith("_"))) .collect(Collectors.toList()); - listener.onResponse(results); + previewResponse.setDocs(results); + listener.onResponse(previewResponse); } else { List> results = pivot.extractResults(agg, deducedMappings, stats) .map(doc -> { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml index 44a8225ef24..dbcacb06379 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml @@ -98,6 +98,11 @@ setup: - match: { preview.2.avg_response: 42.0 } - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" } - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" } + - match: { mappings.properties.airline.type: "keyword" } + - match: { mappings.properties.by-hour.type: "date" } + - match: { mappings.properties.avg_response.type: "double" } + - match: { mappings.properties.time\.max.type: "date" } + - match: { mappings.properties.time\.min.type: "date" } - do: ingest.put_pipeline: @@ -141,6 +146,9 @@ setup: - match: { preview.2.by-hour: 1487379600000 } - match: { preview.2.avg_response: 42.0 } - match: { preview.2.my_field: 42 } + - match: { mappings.properties.airline.type: "keyword" } + - match: { mappings.properties.by-hour.type: "date" } + - match: { mappings.properties.avg_response.type: "double" } --- "Test preview transform with invalid config":