From 292e0f6fb76a1e280580cb4d5798a7cc876eb8ae Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 4 Feb 2019 16:11:44 -0700 Subject: [PATCH] Deprecate `_type` in simulate pipeline requests (#37949) As mapping types are being removed throughout Elasticsearch, the use of `_type` in pipeline simulation requests is deprecated. Additionally, the default `_type` used if one is not supplied has been changed to `_doc` for consistency with the rest of Elasticsearch. --- .../elasticsearch/client/IngestClientIT.java | 1 - .../IngestClientDocumentationIT.java | 8 +-- .../ingest/apis/simulate-pipeline.asciidoc | 4 -- .../processors/date-index-name.asciidoc | 2 +- .../reference/ingest/processors/grok.asciidoc | 4 +- .../ingest/SimulatePipelineRequest.java | 12 +++- .../SimulatePipelineRequestParsingTests.java | 63 ++++++++++++++----- 7 files changed, 67 insertions(+), 27 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 84bf43ab019..1c10f65fb36 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -130,7 +130,6 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase { { builder.startObject() .field("_index", "index") - .field("_type", "doc") .field("_id", "doc_" + 1) .startObject("_source").field("foo", "rab_" + 1).field("rank", rankValue).endObject() .endObject(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index 00bee27807f..df27b1f1c1a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -296,8 +296,8 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( @@ -353,8 +353,8 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index d4f043e8021..deb464eac7a 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -65,7 +65,6 @@ POST _ingest/pipeline/_simulate "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -73,7 +72,6 @@ POST _ingest/pipeline/_simulate }, { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "rab" @@ -158,7 +156,6 @@ POST _ingest/pipeline/_simulate?verbose "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -166,7 +163,6 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "rab" diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index fcece261bd4..6dd54dab056 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -112,7 +112,7 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", - "_type" : "_type", + "_type" : "_doc", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index b266879e40b..f6f5fb3c928 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -193,7 +193,7 @@ response: "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { @@ -254,7 +254,7 @@ POST _ingest/pipeline/_simulate "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 4c2736e3d86..eb15b56db31 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -43,6 +46,9 @@ import java.util.Objects; public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject { + private static final Logger logger = LogManager.getLogger(SimulatePipelineRequest.class); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + private String id; private boolean verbose; private BytesReference source; @@ -178,8 +184,12 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent dataMap, Fields.SOURCE); String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"); + if (dataMap.containsKey(MetaData.TYPE.getFieldName())) { + deprecationLogger.deprecatedAndMaybeLog("simulate_pipeline_with_types", + "[types removal] specifying _type in pipeline simulation requests is deprecated"); + } String type = ConfigurationUtils.readStringOrIntProperty(null, null, - dataMap, MetaData.TYPE.getFieldName(), "_type"); + dataMap, MetaData.TYPE.getFieldName(), "_doc"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 1711d168910..8e313e7cdbb 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -19,15 +19,6 @@ package org.elasticsearch.action.ingest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.CompoundProcessor; import org.elasticsearch.ingest.IngestDocument; @@ -38,6 +29,15 @@ import org.elasticsearch.ingest.TestProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields; import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID; import static org.elasticsearch.ingest.IngestDocument.MetaData.ID; @@ -67,7 +67,15 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { when(ingestService.getProcessorFactories()).thenReturn(registry); } - public void testParseUsingPipelineStore() throws Exception { + public void testParseUsingPipelineStoreNoType() throws Exception { + innerTestParseUsingPipelineStore(false); + } + + public void testParseUsingPipelineStoreWithType() throws Exception { + innerTestParseUsingPipelineStore(true); + } + + private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -80,7 +88,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { String type = randomAlphaOfLengthBetween(1, 10); String id = randomAlphaOfLengthBetween(1, 10); doc.put(INDEX.getFieldName(), index); - doc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + doc.put(TYPE.getFieldName(), type); + } doc.put(ID.getFieldName(), id); String fieldName = randomAlphaOfLengthBetween(1, 10); String fieldValue = randomAlphaOfLengthBetween(1, 10); @@ -88,7 +98,11 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { docs.add(doc); Map expectedDoc = new HashMap<>(); expectedDoc.put(INDEX.getFieldName(), index); - expectedDoc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + expectedDoc.put(TYPE.getFieldName(), type); + } else { + expectedDoc.put(TYPE.getFieldName(), "_doc"); + } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); expectedDocs.add(expectedDoc); @@ -111,9 +125,20 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1)); + if (useExplicitType) { + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); + } } - public void testParseWithProvidedPipeline() throws Exception { + public void testParseWithProvidedPipelineNoType() throws Exception { + innerTestParseWithProvidedPipeline(false); + } + + public void testParseWithProvidedPipelineWithType() throws Exception { + innerTestParseWithProvidedPipeline(true); + } + + private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -135,6 +160,14 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { ); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); + } else if (field == TYPE) { + if (useExplicitType) { + String value = randomAlphaOfLengthBetween(1, 10); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } else { + expectedDoc.put(field.getFieldName(), "_doc"); + } } else { if (randomBoolean()) { String value = randomAlphaOfLengthBetween(1, 10); @@ -191,7 +224,6 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { Map expectedDocument = expectedDocsIterator.next(); Map metadataMap = ingestDocument.extractMetadata(); assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName()))); assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName()))); assertThat(metadataMap.get(VERSION), equalTo(expectedDocument.get(VERSION.getFieldName()))); @@ -202,6 +234,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors)); + if (useExplicitType) { + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); + } } public void testNullPipelineId() {