diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index e316759e041..aba7cd69359 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -19,14 +19,18 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.InternalSettingsPlugin; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; @@ -39,6 +43,12 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC import static org.hamcrest.Matchers.hasSize; public class DeleteByQueryBasicTests extends ReindexTestCase { + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(InternalSettingsPlugin.class); + return plugins; + } public void testBasics() throws Exception { indexRandom(true, @@ -237,4 +247,26 @@ public class DeleteByQueryBasicTests extends ReindexTestCase { assertThat(request.get(), matcher().deleted(5).slices(hasSize(5))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } + + /** + * Test delete by query support for filtering by type. This entire feature + * can and should be removed when we drop support for types index with + * multiple types from core. + */ + public void testFilterByType() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types + indexRandom(true, + client().prepareIndex("test", "test1", "1").setSource("foo", "a"), + client().prepareIndex("test", "test2", "2").setSource("foo", "a"), + client().prepareIndex("test", "test2", "3").setSource("foo", "b")); + + assertHitCount(client().prepareSearch("test").setSize(0).get(), 3); + + // Deletes doc of the type "type2" that also matches foo:a + DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true); + builder.source().setTypes("test2"); + assertThat(builder.get(), matcher().deleted(1)); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 2); + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java index 218a6b9eed4..14eb9245939 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java @@ -25,14 +25,19 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.test.InternalSettingsPlugin; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.index.query.QueryBuilders.idsQuery; +import static org.elasticsearch.index.query.QueryBuilders.typeQuery; import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -42,7 +47,8 @@ import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; /** - * Index-by-search tests for parent/child. + * Reindex tests for legacy parent/child. Tests for the new {@code join} + * field are in a qa project. */ public class ReindexParentChildTests extends ReindexTestCase { QueryBuilder findsCountry; @@ -59,6 +65,7 @@ public class ReindexParentChildTests extends ReindexTestCase { final List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(ParentJoinPlugin.class); plugins.add(InternalSettingsPlugin.class); + plugins.add(CustomScriptPlugin.class); return Collections.unmodifiableList(plugins); } @@ -70,7 +77,7 @@ public class ReindexParentChildTests extends ReindexTestCase { public void testParentChild() throws Exception { createParentChildIndex("source"); createParentChildIndex("dest"); - createParentChildDocs("source"); + createParentChildDocs("source", true); // Copy parent to the new index ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCountry).refresh(true); @@ -101,9 +108,32 @@ public class ReindexParentChildTests extends ReindexTestCase { "make-believe"); } + /** + * Tests for adding the {@code _parent} via script and adding *both* {@code _parent} and {@code _routing} values via scripts. + */ + public void testScriptAddsParent() throws Exception { + assertAcked(client().admin().indices().prepareCreate("source") + .setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types + + createParentChildIndex("dest"); + createParentChildDocs("source", false); + + ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(typeQuery("country")).refresh(true); + assertThat(copy.get(), matcher().created(1)); + copy = reindex().source("source").destination("dest").filter(typeQuery("city")) + .script(mockScript("ctx._parent='united states'")).refresh(true); + assertThat(copy.get(), matcher().created(1)); + assertSearchHits(client().prepareSearch("dest").setQuery(findsCity).get(), "pittsburgh"); + + copy = reindex().source("source").destination("dest").filter(typeQuery("neighborhood")) + .script(mockScript("ctx._parent='pittsburgh';ctx._routing='united states'")).refresh(true); + assertThat(copy.get(), matcher().created(1)); + assertSearchHits(client().prepareSearch("dest").setQuery(findsNeighborhood).get(), "make-believe"); + } + public void testErrorMessageWhenBadParentChild() throws Exception { createParentChildIndex("source"); - createParentChildDocs("source"); + createParentChildDocs("source", true); ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCity); final BulkByScrollResponse response = copy.get(); @@ -119,25 +149,55 @@ public class ReindexParentChildTests extends ReindexTestCase { */ private void createParentChildIndex(String indexName) throws Exception { CreateIndexRequestBuilder create = client().admin().indices().prepareCreate(indexName); - create.setSettings("index.version.created", Version.V_5_6_0.id); + create.setSettings("index.version.created", Version.V_5_6_0.id); // allows for multiple types create.addMapping("city", "{\"_parent\": {\"type\": \"country\"}}", XContentType.JSON); create.addMapping("neighborhood", "{\"_parent\": {\"type\": \"city\"}}", XContentType.JSON); assertAcked(create); ensureGreen(); } - private void createParentChildDocs(String indexName) throws Exception { - indexRandom(true, client().prepareIndex(indexName, "country", "united states").setSource("foo", "bar"), - client().prepareIndex(indexName, "city", "pittsburgh").setParent("united states").setSource("foo", "bar"), - client().prepareIndex(indexName, "neighborhood", "make-believe").setParent("pittsburgh") - .setSource("foo", "bar").setRouting("united states")); + private void createParentChildDocs(String indexName, boolean addParents) throws Exception { + indexRandom(true, + client().prepareIndex(indexName, "country", "united states") + .setSource("foo", "bar"), + client().prepareIndex(indexName, "city", "pittsburgh") + .setParent(addParents ? "united states" : null) + .setSource("foo", "bar"), + client().prepareIndex(indexName, "neighborhood", "make-believe") + .setParent(addParents ? "pittsburgh" : null) + .setRouting(addParents ? "united states" : null) + .setSource("foo", "bar")); findsCountry = idsQuery("country").addIds("united states"); findsCity = hasParentQuery("country", findsCountry, false); findsNeighborhood = hasParentQuery("city", findsCity, false); - // Make sure we built the parent/child relationship - assertSearchHits(client().prepareSearch(indexName).setQuery(findsCity).get(), "pittsburgh"); - assertSearchHits(client().prepareSearch(indexName).setQuery(findsNeighborhood).get(), "make-believe"); + if (addParents) { + // Make sure we built the parent/child relationship + assertSearchHits(client().prepareSearch(indexName).setQuery(findsCity).get(), "pittsburgh"); + assertSearchHits(client().prepareSearch(indexName).setQuery(findsNeighborhood).get(), "make-believe"); + } + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + + scripts.put("ctx._parent='united states'", vars -> { + Map ctx = (Map) vars.get("ctx"); + ctx.put("_parent", "united states"); + return null; + }); + scripts.put("ctx._parent='pittsburgh';ctx._routing='united states'", vars -> { + Map ctx = (Map) vars.get("ctx"); + ctx.put("_parent", "pittsburgh"); + ctx.put("_routing", "united states"); + return null; + }); + + return scripts; + } } } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yml deleted file mode 100644 index 4ed279a0165..00000000000 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yml +++ /dev/null @@ -1,79 +0,0 @@ ---- -"Delete by type": - - do: - indices.create: - index: test - body: - settings: - mapping.single_type: false - - - do: - index: - index: test - type: t1 - id: 1 - body: { foo: bar } - - do: - index: - index: test - type: t1 - id: 2 - body: { foo: bar } - - do: - index: - index: test - type: t2 - id: 1 - body: { foo: bar } - - do: - index: - index: test - type: t2 - id: 2 - body: { foo: bar } - - do: - index: - index: test - type: t2 - id: 3 - body: { foo: baz } - - do: - indices.refresh: {} - - do: - count: - index: test - type: t2 - - - match: {count: 3} - - - do: - delete_by_query: - index: test - type: t2 - body: - query: - match: - foo: bar - - - is_false: timed_out - - match: {deleted: 2} - - is_false: created - - is_false: updated - - match: {version_conflicts: 0} - - match: {batches: 1} - - match: {failures: []} - - match: {noops: 0} - - match: {throttled_millis: 0} - - gte: { took: 0 } - - is_false: task - - - do: - indices.refresh: {} - - - do: - count: - index: test - type: t2 - - - match: {count: 1} - diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml index ba30ead5202..8fda091d80d 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml +++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml @@ -81,60 +81,6 @@ user: blort - match: { hits.total: 1 } ---- -"Add new parent": - - do: - indices.create: - index: new_twitter - body: - settings: - mapping.single_type: false - mappings: - tweet: - _parent: { type: "user" } - - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - index: - index: new_twitter - type: user - id: kimchy - body: { "name": "kimchy" } - - do: - indices.refresh: {} - - - do: - reindex: - refresh: true - body: - source: - index: twitter - dest: - index: new_twitter - script: - lang: painless - source: ctx._parent = ctx._source.user - - match: {created: 1} - - match: {noops: 0} - - - do: - search: - index: new_twitter - body: - query: - has_parent: - parent_type: user - query: - match: - name: kimchy - - match: { hits.total: 1 } - - match: { hits.hits.0._source.user: kimchy } - --- "Add routing": - do: @@ -182,63 +128,6 @@ routing: foo - match: { _routing: foo } ---- -"Add routing and parent": - - do: - indices.create: - index: new_twitter - body: - settings: - mapping.single_type: false - mappings: - tweet: - _parent: { type: "user" } - - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - index: - index: new_twitter - type: user - id: kimchy - body: { "name": "kimchy" } - routing: cat - - do: - indices.refresh: {} - - - do: - reindex: - refresh: true - body: - source: - index: twitter - dest: - index: new_twitter - script: - lang: painless - source: ctx._parent = ctx._source.user; ctx._routing = "cat" - - match: {created: 1} - - match: {noops: 0} - - - do: - search: - index: new_twitter - routing: cat - body: - query: - has_parent: - parent_type: user - query: - match: - name: kimchy - - match: { hits.total: 1 } - - match: { hits.hits.0._source.user: kimchy } - - match: { hits.hits.0._routing: cat } - --- "Noop one doc": - do: diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml new file mode 100644 index 00000000000..496c13ec9b4 --- /dev/null +++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml @@ -0,0 +1,148 @@ +setup: + - do: + indices.create: + index: source + body: + mappings: + doc: + properties: + join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + + - do: + indices.create: + index: dest + body: + mappings: + doc: + properties: + join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + + - do: + index: + index: source + type: doc + id: 1 + body: { "join_field": { "name": "parent" } } + + - do: + index: + index: source + type: doc + id: 2 + routing: 1 + body: { "join_field": { "name": "child", "parent": "1" } } + + - do: + index: + index: source + type: doc + id: 3 + routing: 1 + body: { "join_field": { "name": "grand_child", "parent": "2" } } + + - do: + indices.refresh: {} + + +--- +"Reindex with parent join field": + - do: + reindex: + refresh: true + body: + source: + index: source + dest: + index: dest + - match: {created: 3} + + - do: + search: + index: dest + body: + query: + parent_id: + type: child + id: 1 + - match: {hits.total: 1} + - match: {hits.hits.0._id: "2"} + + - do: + search: + index: dest + body: + query: + has_parent: + parent_type: child + query: + parent_id: + type: child + id: 1 + - match: {hits.total: 1} + - match: {hits.hits.0._id: "3"} + + # Make sure reindex closed all the scroll contexts + - do: + indices.stats: + index: source + metric: search + - match: {indices.source.total.search.open_contexts: 0} + + +--- +"Reindex from remote with parent join field": + - skip: + reason: Temporarily broken. See https://github.com/elastic/elasticsearch/issues/25363 + version: all + # Fetch the http host. We use the host of the master because we know there will always be a master. + - do: + cluster.state: {} + - set: { master_node: master } + - do: + nodes.info: + metric: [ http ] + - is_true: nodes.$master.http.publish_address + - set: {nodes.$master.http.publish_address: host} + - do: + reindex: + refresh: true + body: + source: + remote: + host: http://${host} + index: source + dest: + index: dest + - match: {created: 3} + + - do: + search: + index: dest + body: + query: + parent_id: + type: child + id: 1 + - match: {hits.total: 1} + - match: {hits.hits.0._id: "2"} + + - do: + search: + index: dest + body: + query: + has_parent: + parent_type: child + query: + parent_id: + type: child + id: 1 + - match: {hits.total: 1} + - match: {hits.hits.0._id: "3"} + + # Make sure reindex closed all the scroll contexts + - do: + indices.stats: + index: source + metric: search + - match: {indices.source.total.search.open_contexts: 0} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yml deleted file mode 100644 index 81e142c9195..00000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yml +++ /dev/null @@ -1,79 +0,0 @@ ---- -"Reindex from remote with parent/child": - - do: - indices.create: - index: source - body: - settings: - mapping.single_type: false - mappings: - foo: {} - bar: - _parent: - type: foo - - do: - indices.create: - index: dest - body: - settings: - mapping.single_type: false - mappings: - foo: {} - bar: - _parent: - type: foo - - do: - index: - index: source - type: foo - id: 1 - body: { "text": "test" } - - do: - index: - index: source - type: bar - id: 1 - parent: 1 - body: { "text": "test2" } - - do: - indices.refresh: {} - - # Fetch the http host. We use the host of the master because we know there will always be a master. - - do: - cluster.state: {} - - set: { master_node: master } - - do: - nodes.info: - metric: [ http ] - - is_true: nodes.$master.http.publish_address - - set: {nodes.$master.http.publish_address: host} - - do: - reindex: - refresh: true - body: - source: - remote: - host: http://${host} - index: source - dest: - index: dest - - match: {created: 2} - - - do: - search: - index: dest - body: - query: - has_parent: - parent_type: foo - query: - match: - text: test - - match: {hits.total: 1} - - # Make sure reindex closed all the scroll contexts - - do: - indices.stats: - index: source - metric: search - - match: {indices.source.total.search.open_contexts: 0}