Remove `index.mapping.single_type=false` from reindex tests (#25365)

* Remove the setting from the yml tests and replace with tests using
`join` field. We can't use the setting in yml tests without lots of
backflips but we have `ReindexParentChildTests` for the coverage.
There weren't tests for `join` field with reindex before this. Adding
these tests discovered #25363.
* Remove the setting from `ReindexParentChildTests` and replace with
`index.version.created=V_5_6_0`. This test can be entirely removed
when legacy parent/child support is dropped from core.
* Port the yml tests that set _parent into integ tests so they
can set the index created version. These tests can be removed
when we drop support for _parent in core.
* Port a delete-by-query test for filtering based on type to an
`ESIntegTestCase` so it can use `index.version.created=5.6.0` to
setup documents of multiple types. This whole feature can be dropped
when we no longer support multiple types per index.

Relates to #24961

modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java

@@ -19,14 +19,18 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
@@ -39,6 +43,12 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
 import static org.hamcrest.Matchers.hasSize;
 
 public class DeleteByQueryBasicTests extends ReindexTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(InternalSettingsPlugin.class);
+        return plugins;
+    }
 
     public void testBasics() throws Exception {
         indexRandom(true,
@@ -237,4 +247,26 @@ public class DeleteByQueryBasicTests extends ReindexTestCase {
         assertThat(request.get(), matcher().deleted(5).slices(hasSize(5)));
         assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
     }
+
+    /**
+     * Test delete by query support for filtering by type. This entire feature
+     * can and should be removed when we drop support for types index with
+     * multiple types from core.
+     */
+    public void testFilterByType() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types
+        indexRandom(true,
+                client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
+                client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
+                client().prepareIndex("test", "test2", "3").setSource("foo", "b"));
+
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
+
+        // Deletes doc of the type "type2" that also matches foo:a
+        DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
+        builder.source().setTypes("test2");
+        assertThat(builder.get(), matcher().deleted(1));
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
+    }
 }

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java

@@ -25,14 +25,19 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.typeQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
@@ -42,7 +47,8 @@ import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
 
 /**
- * Index-by-search tests for parent/child.
+ * Reindex tests for legacy parent/child. Tests for the new {@code join}
+ * field are in a qa project.
  */
 public class ReindexParentChildTests extends ReindexTestCase {
     QueryBuilder findsCountry;
@@ -59,6 +65,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
         final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
         plugins.add(ParentJoinPlugin.class);
         plugins.add(InternalSettingsPlugin.class);
+        plugins.add(CustomScriptPlugin.class);
         return Collections.unmodifiableList(plugins);
     }
 
@@ -70,7 +77,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
     public void testParentChild() throws Exception {
         createParentChildIndex("source");
         createParentChildIndex("dest");
-        createParentChildDocs("source");
+        createParentChildDocs("source", true);
 
         // Copy parent to the new index
         ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCountry).refresh(true);
@@ -101,9 +108,32 @@ public class ReindexParentChildTests extends ReindexTestCase {
                 "make-believe");
     }
 
+    /**
+     * Tests for adding the {@code _parent} via script and adding *both* {@code _parent} and {@code _routing} values via scripts.
+     */
+    public void testScriptAddsParent() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("source")
+            .setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types
+
+        createParentChildIndex("dest");
+        createParentChildDocs("source", false);
+
+        ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(typeQuery("country")).refresh(true);
+        assertThat(copy.get(), matcher().created(1));
+        copy = reindex().source("source").destination("dest").filter(typeQuery("city"))
+                .script(mockScript("ctx._parent='united states'")).refresh(true);
+        assertThat(copy.get(), matcher().created(1));
+        assertSearchHits(client().prepareSearch("dest").setQuery(findsCity).get(), "pittsburgh");
+
+        copy = reindex().source("source").destination("dest").filter(typeQuery("neighborhood"))
+                .script(mockScript("ctx._parent='pittsburgh';ctx._routing='united states'")).refresh(true);
+        assertThat(copy.get(), matcher().created(1));
+        assertSearchHits(client().prepareSearch("dest").setQuery(findsNeighborhood).get(), "make-believe");
+    }
+
     public void testErrorMessageWhenBadParentChild() throws Exception {
         createParentChildIndex("source");
-        createParentChildDocs("source");
+        createParentChildDocs("source", true);
 
         ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCity);
         final BulkByScrollResponse response = copy.get();
@@ -119,25 +149,55 @@ public class ReindexParentChildTests extends ReindexTestCase {
      */
     private void createParentChildIndex(String indexName) throws Exception {
         CreateIndexRequestBuilder create = client().admin().indices().prepareCreate(indexName);
-        create.setSettings("index.version.created", Version.V_5_6_0.id);
+        create.setSettings("index.version.created", Version.V_5_6_0.id); // allows for multiple types
         create.addMapping("city", "{\"_parent\": {\"type\": \"country\"}}", XContentType.JSON);
         create.addMapping("neighborhood", "{\"_parent\": {\"type\": \"city\"}}", XContentType.JSON);
         assertAcked(create);
         ensureGreen();
     }
 
-    private void createParentChildDocs(String indexName) throws Exception {
+    private void createParentChildDocs(String indexName, boolean addParents) throws Exception {
-        indexRandom(true, client().prepareIndex(indexName, "country", "united states").setSource("foo", "bar"),
+        indexRandom(true,
-                client().prepareIndex(indexName, "city", "pittsburgh").setParent("united states").setSource("foo", "bar"),
+                client().prepareIndex(indexName, "country", "united states")
-                client().prepareIndex(indexName, "neighborhood", "make-believe").setParent("pittsburgh")
+                        .setSource("foo", "bar"),
-                        .setSource("foo", "bar").setRouting("united states"));
+                client().prepareIndex(indexName, "city", "pittsburgh")
+                        .setParent(addParents ? "united states" : null)
+                        .setSource("foo", "bar"),
+                client().prepareIndex(indexName, "neighborhood", "make-believe")
+                        .setParent(addParents ? "pittsburgh" : null)
+                        .setRouting(addParents ? "united states" : null)
+                        .setSource("foo", "bar"));
 
         findsCountry = idsQuery("country").addIds("united states");
         findsCity = hasParentQuery("country", findsCountry, false);
         findsNeighborhood = hasParentQuery("city", findsCity, false);
 
+        if (addParents) {
             // Make sure we built the parent/child relationship
             assertSearchHits(client().prepareSearch(indexName).setQuery(findsCity).get(), "pittsburgh");
             assertSearchHits(client().prepareSearch(indexName).setQuery(findsNeighborhood).get(), "make-believe");
         }
     }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        @SuppressWarnings("unchecked")
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+            scripts.put("ctx._parent='united states'", vars -> {
+                Map<String, String> ctx = (Map<String, String>) vars.get("ctx");
+                ctx.put("_parent", "united states");
+                return null;
+            });
+            scripts.put("ctx._parent='pittsburgh';ctx._routing='united states'", vars -> {
+                Map<String, String> ctx = (Map<String, String>) vars.get("ctx");
+                ctx.put("_parent", "pittsburgh");
+                ctx.put("_routing", "united states");
+                return null;
+            });
+
+            return scripts;
+        }
+    }
+}

modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yml

@@ -1,79 +0,0 @@
----
-"Delete by type":
-  - do:
-      indices.create:
-          index: test
-          body:
-              settings:
-                mapping.single_type: false
-
-  - do:
-      index:
-          index:  test
-          type:   t1
-          id:     1
-          body:   { foo: bar }
-  - do:
-      index:
-          index:  test
-          type:   t1
-          id:     2
-          body:   { foo: bar }
-  - do:
-      index:
-          index:  test
-          type:   t2
-          id:     1
-          body:   { foo: bar }
-  - do:
-      index:
-          index:  test
-          type:   t2
-          id:     2
-          body:   { foo: bar }
-  - do:
-      index:
-          index:  test
-          type:   t2
-          id:     3
-          body:   { foo: baz }
-  - do:
-      indices.refresh: {}
-  - do:
-      count:
-        index: test
-        type: t2
-
-  - match: {count: 3}
-
-  - do:
-      delete_by_query:
-        index: test
-        type: t2
-        body:
-          query:
-            match:
-              foo: bar
-
-  - is_false: timed_out
-  - match: {deleted: 2}
-  - is_false: created
-  - is_false: updated
-  - match: {version_conflicts: 0}
-  - match: {batches: 1}
-  - match: {failures: []}
-  - match: {noops: 0}
-  - match: {throttled_millis: 0}
-  - gte: { took: 0 }
-  - is_false: task
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      count:
-        index: test
-        type: t2
-
-  - match: {count: 1}
-

qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml

@@ -81,60 +81,6 @@
               user: blort
   - match: { hits.total: 1 }
 
----
-"Add new parent":
-  - do:
-      indices.create:
-          index: new_twitter
-          body:
-            settings:
-              mapping.single_type: false
-            mappings:
-              tweet:
-                _parent: { type: "user" }
-
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  new_twitter
-        type:   user
-        id:     kimchy
-        body:   { "name": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            source: ctx._parent = ctx._source.user
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index: new_twitter
-        body:
-          query:
-            has_parent:
-              parent_type: user
-              query:
-                match:
-                  name: kimchy
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._source.user: kimchy }
-
 ---
 "Add routing":
   - do:
@@ -182,63 +128,6 @@
         routing: foo
   - match: { _routing: foo }
 
----
-"Add routing and parent":
-  - do:
-      indices.create:
-          index: new_twitter
-          body:
-            settings:
-              mapping.single_type: false
-            mappings:
-              tweet:
-                _parent: { type: "user" }
-
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:  new_twitter
-        type:   user
-        id:     kimchy
-        body:   { "name": "kimchy" }
-        routing: cat
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-          script:
-            lang: painless
-            source: ctx._parent = ctx._source.user; ctx._routing = "cat"
-  - match: {created: 1}
-  - match: {noops: 0}
-
-  - do:
-      search:
-        index:   new_twitter
-        routing: cat
-        body:
-          query:
-            has_parent:
-              parent_type: user
-              query:
-                match:
-                  name: kimchy
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._source.user: kimchy }
-  - match: { hits.hits.0._routing: cat }
-
 ---
 "Noop one doc":
   - do:

qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml

@@ -0,0 +1,148 @@
+setup:
+  - do:
+      indices.create:
+          index: source
+          body:
+            mappings:
+              doc:
+                properties:
+                  join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
+
+  - do:
+      indices.create:
+          index: dest
+          body:
+            mappings:
+              doc:
+                properties:
+                  join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
+
+  - do:
+      index:
+        index: source
+        type:  doc
+        id:    1
+        body:  { "join_field": { "name": "parent" } }
+
+  - do:
+      index:
+        index: source
+        type:  doc
+        id:    2
+        routing: 1
+        body:  { "join_field": { "name": "child", "parent": "1" } }
+
+  - do:
+      index:
+        index: source
+        type:  doc
+        id:    3
+        routing: 1
+        body:  { "join_field": { "name": "grand_child", "parent": "2" } }
+
+  - do:
+      indices.refresh: {}
+
+
+---
+"Reindex with parent join field":
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: source
+          dest:
+            index: dest
+  - match: {created: 3}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            parent_id:
+              type: child
+              id: 1
+  - match: {hits.total: 1}
+  - match: {hits.hits.0._id: "2"}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            has_parent:
+              parent_type: child
+              query:
+                parent_id:
+                  type: child
+                  id: 1
+  - match: {hits.total: 1}
+  - match: {hits.hits.0._id: "3"}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
+
+---
+"Reindex from remote with parent join field":
+  - skip:
+      reason: Temporarily broken. See https://github.com/elastic/elasticsearch/issues/25363
+      version: all
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 3}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            parent_id:
+              type: child
+              id: 1
+  - match: {hits.total: 1}
+  - match: {hits.hits.0._id: "2"}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            has_parent:
+              parent_type: child
+              query:
+                parent_id:
+                  type: child
+                  id: 1
+  - match: {hits.total: 1}
+  - match: {hits.hits.0._id: "3"}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}

qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yml

@@ -1,79 +0,0 @@
----
-"Reindex from remote with parent/child":
-  - do:
-      indices.create:
-          index: source
-          body:
-              settings:
-                mapping.single_type: false
-              mappings:
-                foo: {}
-                bar:
-                  _parent:
-                    type:  foo
-  - do:
-      indices.create:
-          index: dest
-          body:
-              settings:
-                mapping.single_type: false
-              mappings:
-                foo: {}
-                bar:
-                  _parent:
-                    type:  foo
-  - do:
-      index:
-        index:   source
-        type:    foo
-        id:      1
-        body:    { "text": "test" }
-  - do:
-      index:
-        index:   source
-        type:    bar
-        id:      1
-        parent:  1
-        body:    { "text": "test2" }
-  - do:
-      indices.refresh: {}
-
-  # Fetch the http host. We use the host of the master because we know there will always be a master.
-  - do:
-      cluster.state: {}
-  - set: { master_node: master }
-  - do:
-      nodes.info:
-        metric: [ http ]
-  - is_true: nodes.$master.http.publish_address
-  - set: {nodes.$master.http.publish_address: host}
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            remote:
-              host: http://${host}
-            index: source
-          dest:
-            index: dest
-  - match: {created: 2}
-
-  - do:
-      search:
-        index: dest
-        body:
-          query:
-            has_parent:
-              parent_type: foo
-              query:
-                match:
-                  text: test
-  - match: {hits.total: 1}
-
-  # Make sure reindex closed all the scroll contexts
-  - do:
-      indices.stats:
-        index: source
-        metric: search
-  - match: {indices.source.total.search.open_contexts: 0}