honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove `index.mapping.single_type=false` from reindex tests (#25365) 

* Remove the setting from the yml tests and replace with tests using
`join` field. We can't use the setting in yml tests without lots of
backflips but we have `ReindexParentChildTests` for the coverage.
There weren't tests for `join` field with reindex before this. Adding
these tests discovered #25363.
* Remove the setting from `ReindexParentChildTests` and replace with
`index.version.created=V_5_6_0`. This test can be entirely removed
when legacy parent/child support is dropped from core.
* Port the yml tests that set _parent into integ tests so they
can set the index created version. These tests can be removed
when we drop support for _parent in core.
* Port a delete-by-query test for filtering based on type to an
`ESIntegTestCase` so it can use `index.version.created=5.6.0` to
setup documents of multiple types. This whole feature can be dropped
when we no longer support multiple types per index.

Relates to #24961
This commit is contained in:
Nik Everett 2017-06-23 17:14:59 -04:00 committed by GitHub
parent e82be00890
commit da0b991331
6 changed files with 253 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
View File

@ -19,14 +19,18 @@
package org.elasticsearch.index.reindex;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.InternalSettingsPlugin;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
@ -39,6 +43,12 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
import static org.hamcrest.Matchers.hasSize;
public class DeleteByQueryBasicTests extends ReindexTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
plugins.add(InternalSettingsPlugin.class);
return plugins;
}
public void testBasics() throws Exception {
indexRandom(true,
@ -237,4 +247,26 @@ public class DeleteByQueryBasicTests extends ReindexTestCase {
assertThat(request.get(), matcher().deleted(5).slices(hasSize(5)));
assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
}
/**
* Test delete by query support for filtering by type. This entire feature
* can and should be removed when we drop support for types index with
* multiple types from core.
*/
public void testFilterByType() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types
indexRandom(true,
client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
client().prepareIndex("test", "test2", "3").setSource("foo", "b"));
assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
// Deletes doc of the type "type2" that also matches foo:a
DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
builder.source().setTypes("test2");
assertThat(builder.get(), matcher().deleted(1));
assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
}
}

84
modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
View File

@ -25,14 +25,19 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.join.ParentJoinPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.test.InternalSettingsPlugin;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import static org.elasticsearch.index.query.QueryBuilders.idsQuery;
import static org.elasticsearch.index.query.QueryBuilders.typeQuery;
import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
@ -42,7 +47,8 @@ import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.instanceOf;
/**
* Index-by-search tests for parent/child.
* Reindex tests for legacy parent/child. Tests for the new {@code join}
* field are in a qa project.
*/
public class ReindexParentChildTests extends ReindexTestCase {
QueryBuilder findsCountry;
@ -59,6 +65,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
plugins.add(ParentJoinPlugin.class);
plugins.add(InternalSettingsPlugin.class);
plugins.add(CustomScriptPlugin.class);
return Collections.unmodifiableList(plugins);
}
@ -70,7 +77,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
public void testParentChild() throws Exception {
createParentChildIndex("source");
createParentChildIndex("dest");
createParentChildDocs("source");
createParentChildDocs("source", true);
// Copy parent to the new index
ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCountry).refresh(true);
@ -101,9 +108,32 @@ public class ReindexParentChildTests extends ReindexTestCase {
"make-believe");
}
/**
* Tests for adding the {@code _parent} via script and adding *both* {@code _parent} and {@code _routing} values via scripts.
*/
public void testScriptAddsParent() throws Exception {
assertAcked(client().admin().indices().prepareCreate("source")
.setSettings("index.version.created", Version.V_5_6_0.id)); // allows for multiple types
createParentChildIndex("dest");
createParentChildDocs("source", false);
ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(typeQuery("country")).refresh(true);
assertThat(copy.get(), matcher().created(1));
copy = reindex().source("source").destination("dest").filter(typeQuery("city"))
.script(mockScript("ctx._parent='united states'")).refresh(true);
assertThat(copy.get(), matcher().created(1));
assertSearchHits(client().prepareSearch("dest").setQuery(findsCity).get(), "pittsburgh");
copy = reindex().source("source").destination("dest").filter(typeQuery("neighborhood"))
.script(mockScript("ctx._parent='pittsburgh';ctx._routing='united states'")).refresh(true);
assertThat(copy.get(), matcher().created(1));
assertSearchHits(client().prepareSearch("dest").setQuery(findsNeighborhood).get(), "make-believe");
}
public void testErrorMessageWhenBadParentChild() throws Exception {
createParentChildIndex("source");
createParentChildDocs("source");
createParentChildDocs("source", true);
ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCity);
final BulkByScrollResponse response = copy.get();
@ -119,25 +149,55 @@ public class ReindexParentChildTests extends ReindexTestCase {
*/
private void createParentChildIndex(String indexName) throws Exception {
CreateIndexRequestBuilder create = client().admin().indices().prepareCreate(indexName);
create.setSettings("index.version.created", Version.V_5_6_0.id);
create.setSettings("index.version.created", Version.V_5_6_0.id); // allows for multiple types
create.addMapping("city", "{\"_parent\": {\"type\": \"country\"}}", XContentType.JSON);
create.addMapping("neighborhood", "{\"_parent\": {\"type\": \"city\"}}", XContentType.JSON);
assertAcked(create);
ensureGreen();
}
private void createParentChildDocs(String indexName) throws Exception {
indexRandom(true, client().prepareIndex(indexName, "country", "united states").setSource("foo", "bar"),
client().prepareIndex(indexName, "city", "pittsburgh").setParent("united states").setSource("foo", "bar"),
client().prepareIndex(indexName, "neighborhood", "make-believe").setParent("pittsburgh")
.setSource("foo", "bar").setRouting("united states"));
private void createParentChildDocs(String indexName, boolean addParents) throws Exception {
indexRandom(true,
client().prepareIndex(indexName, "country", "united states")
.setSource("foo", "bar"),
client().prepareIndex(indexName, "city", "pittsburgh")
.setParent(addParents ? "united states" : null)
.setSource("foo", "bar"),
client().prepareIndex(indexName, "neighborhood", "make-believe")
.setParent(addParents ? "pittsburgh" : null)
.setRouting(addParents ? "united states" : null)
.setSource("foo", "bar"));
findsCountry = idsQuery("country").addIds("united states");
findsCity = hasParentQuery("country", findsCountry, false);
findsNeighborhood = hasParentQuery("city", findsCity, false);
// Make sure we built the parent/child relationship
assertSearchHits(client().prepareSearch(indexName).setQuery(findsCity).get(), "pittsburgh");
assertSearchHits(client().prepareSearch(indexName).setQuery(findsNeighborhood).get(), "make-believe");
if (addParents) {
// Make sure we built the parent/child relationship
assertSearchHits(client().prepareSearch(indexName).setQuery(findsCity).get(), "pittsburgh");
assertSearchHits(client().prepareSearch(indexName).setQuery(findsNeighborhood).get(), "make-believe");
}
}
public static class CustomScriptPlugin extends MockScriptPlugin {
@Override
@SuppressWarnings("unchecked")
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
scripts.put("ctx._parent='united states'", vars -> {
Map<String, String> ctx = (Map<String, String>) vars.get("ctx");
ctx.put("_parent", "united states");
return null;
});
scripts.put("ctx._parent='pittsburgh';ctx._routing='united states'", vars -> {
Map<String, String> ctx = (Map<String, String>) vars.get("ctx");
ctx.put("_parent", "pittsburgh");
ctx.put("_routing", "united states");
return null;
});
return scripts;
}
}
}

79
modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yml
View File

@ -1,79 +0,0 @@
---
"Delete by type":
- do:
indices.create:
index: test
body:
settings:
mapping.single_type: false
- do:
index:
index: test
type: t1
id: 1
body: { foo: bar }
- do:
index:
index: test
type: t1
id: 2
body: { foo: bar }
- do:
index:
index: test
type: t2
id: 1
body: { foo: bar }
- do:
index:
index: test
type: t2
id: 2
body: { foo: bar }
- do:
index:
index: test
type: t2
id: 3
body: { foo: baz }
- do:
indices.refresh: {}
- do:
count:
index: test
type: t2
- match: {count: 3}
- do:
delete_by_query:
index: test
type: t2
body:
query:
match:
foo: bar
- is_false: timed_out
- match: {deleted: 2}
- is_false: created
- is_false: updated
- match: {version_conflicts: 0}
- match: {batches: 1}
- match: {failures: []}
- match: {noops: 0}
- match: {throttled_millis: 0}
- gte: { took: 0 }
- is_false: task
- do:
indices.refresh: {}
- do:
count:
index: test
type: t2
- match: {count: 1}

111
qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml
View File

@ -81,60 +81,6 @@
user: blort
- match: { hits.total: 1 }
---
"Add new parent":
- do:
indices.create:
index: new_twitter
body:
settings:
mapping.single_type: false
mappings:
tweet:
_parent: { type: "user" }
- do:
index:
index: twitter
type: tweet
id: 1
body: { "user": "kimchy" }
- do:
index:
index: new_twitter
type: user
id: kimchy
body: { "name": "kimchy" }
- do:
indices.refresh: {}
- do:
reindex:
refresh: true
body:
source:
index: twitter
dest:
index: new_twitter
script:
lang: painless
source: ctx._parent = ctx._source.user
- match: {created: 1}
- match: {noops: 0}
- do:
search:
index: new_twitter
body:
query:
has_parent:
parent_type: user
query:
match:
name: kimchy
- match: { hits.total: 1 }
- match: { hits.hits.0._source.user: kimchy }
---
"Add routing":
- do:
@ -182,63 +128,6 @@
routing: foo
- match: { _routing: foo }
---
"Add routing and parent":
- do:
indices.create:
index: new_twitter
body:
settings:
mapping.single_type: false
mappings:
tweet:
_parent: { type: "user" }
- do:
index:
index: twitter
type: tweet
id: 1
body: { "user": "kimchy" }
- do:
index:
index: new_twitter
type: user
id: kimchy
body: { "name": "kimchy" }
routing: cat
- do:
indices.refresh: {}
- do:
reindex:
refresh: true
body:
source:
index: twitter
dest:
index: new_twitter
script:
lang: painless
source: ctx._parent = ctx._source.user; ctx._routing = "cat"
- match: {created: 1}
- match: {noops: 0}
- do:
search:
index: new_twitter
routing: cat
body:
query:
has_parent:
parent_type: user
query:
match:
name: kimchy
- match: { hits.total: 1 }
- match: { hits.hits.0._source.user: kimchy }
- match: { hits.hits.0._routing: cat }
---
"Noop one doc":
- do:

148
qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml Normal file
View File

@ -0,0 +1,148 @@
setup:
- do:
indices.create:
index: source
body:
mappings:
doc:
properties:
join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
- do:
indices.create:
index: dest
body:
mappings:
doc:
properties:
join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
- do:
index:
index: source
type: doc
id: 1
body: { "join_field": { "name": "parent" } }
- do:
index:
index: source
type: doc
id: 2
routing: 1
body: { "join_field": { "name": "child", "parent": "1" } }
- do:
index:
index: source
type: doc
id: 3
routing: 1
body: { "join_field": { "name": "grand_child", "parent": "2" } }
- do:
indices.refresh: {}
---
"Reindex with parent join field":
- do:
reindex:
refresh: true
body:
source:
index: source
dest:
index: dest
- match: {created: 3}
- do:
search:
index: dest
body:
query:
parent_id:
type: child
id: 1
- match: {hits.total: 1}
- match: {hits.hits.0._id: "2"}
- do:
search:
index: dest
body:
query:
has_parent:
parent_type: child
query:
parent_id:
type: child
id: 1
- match: {hits.total: 1}
- match: {hits.hits.0._id: "3"}
# Make sure reindex closed all the scroll contexts
- do:
indices.stats:
index: source
metric: search
- match: {indices.source.total.search.open_contexts: 0}
---
"Reindex from remote with parent join field":
- skip:
reason: Temporarily broken. See https://github.com/elastic/elasticsearch/issues/25363
version: all
# Fetch the http host. We use the host of the master because we know there will always be a master.
- do:
cluster.state: {}
- set: { master_node: master }
- do:
nodes.info:
metric: [ http ]
- is_true: nodes.$master.http.publish_address
- set: {nodes.$master.http.publish_address: host}
- do:
reindex:
refresh: true
body:
source:
remote:
host: http://${host}
index: source
dest:
index: dest
- match: {created: 3}
- do:
search:
index: dest
body:
query:
parent_id:
type: child
id: 1
- match: {hits.total: 1}
- match: {hits.hits.0._id: "2"}
- do:
search:
index: dest
body:
query:
has_parent:
parent_type: child
query:
parent_id:
type: child
id: 1
- match: {hits.total: 1}
- match: {hits.hits.0._id: "3"}
# Make sure reindex closed all the scroll contexts
- do:
indices.stats:
index: source
metric: search
- match: {indices.source.total.search.open_contexts: 0}

79
qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parentchild.yml
View File

@ -1,79 +0,0 @@
---
"Reindex from remote with parent/child":
- do:
indices.create:
index: source
body:
settings:
mapping.single_type: false
mappings:
foo: {}
bar:
_parent:
type: foo
- do:
indices.create:
index: dest
body:
settings:
mapping.single_type: false
mappings:
foo: {}
bar:
_parent:
type: foo
- do:
index:
index: source
type: foo
id: 1
body: { "text": "test" }
- do:
index:
index: source
type: bar
id: 1
parent: 1
body: { "text": "test2" }
- do:
indices.refresh: {}
# Fetch the http host. We use the host of the master because we know there will always be a master.
- do:
cluster.state: {}
- set: { master_node: master }
- do:
nodes.info:
metric: [ http ]
- is_true: nodes.$master.http.publish_address
- set: {nodes.$master.http.publish_address: host}
- do:
reindex:
refresh: true
body:
source:
remote:
host: http://${host}
index: source
dest:
index: dest
- match: {created: 2}
- do:
search:
index: dest
body:
query:
has_parent:
parent_type: foo
query:
match:
text: test
- match: {hits.total: 1}
# Make sure reindex closed all the scroll contexts
- do:
indices.stats:
index: source
metric: search
- match: {indices.source.total.search.open_contexts: 0}