honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a cluster setting to disallow loading fielddata on _id field (#49166) 

This change adds a dynamic cluster setting named `indices.id_field_data.enabled`.
When set to `false` any attempt to load the fielddata for the `_id` field will fail
with an exception. The default value in this change is set to `false` in order to prevent
fielddata usage on this field for future versions but it will be set to `true` when backporting
to 7x. When the setting is set to true (manually or by default in 7x) the loading will also issue
a deprecation warning since we want to disallow fielddata entirely when https://github.com/elastic/elasticsearch/issues/26472
is implemented.

Closes #43599
This commit is contained in:
Jim Ferenczi 2019-11-27 13:38:09 +01:00 committed by jimczi
parent b236076f88
commit d6445fae4b
36 changed files with 493 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
View File

@ -116,32 +116,32 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/type/1");
doc1.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}");
doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/type/2");
doc2.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}");
doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}");
client().performRequest(doc2);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/type/3");
doc3.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}");
doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}");
client().performRequest(doc3);
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/type/4");
doc4.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}");
client().performRequest(doc4);
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/type/5");
doc5.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}");
client().performRequest(doc5);
}
{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1");
doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}");
doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2");
doc2.setJsonEntity("{\"field\":\"value2\"}");
doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}");
client().performRequest(doc2);
}
@ -159,19 +159,19 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
"}");
client().performRequest(create);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3");
doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}");
doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}");
client().performRequest(doc3);
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4");
doc4.setJsonEntity("{\"field\":\"value2\"}");
doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}");
client().performRequest(doc4);
}
{
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5");
doc5.setJsonEntity("{\"field\":\"value1\"}");
doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}");
client().performRequest(doc5);
Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6");
doc6.setJsonEntity("{\"field\":\"value2\"}");
doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}");
client().performRequest(doc6);
}
@ -194,7 +194,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
"}");
client().performRequest(create);
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1");
doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}");
doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}");
client().performRequest(doc1);
Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases");
@ -232,7 +232,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
assertEquals(1.0f, searchHit.getScore(), 0);
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertEquals(4, searchHit.getSourceAsMap().size());
assertTrue(searchHit.getSourceAsMap().containsKey("type"));
assertTrue(searchHit.getSourceAsMap().containsKey("num"));
assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
@ -257,7 +257,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
assertThat(searchHit.getScore(), greaterThan(0f));
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertEquals(4, searchHit.getSourceAsMap().size());
assertEquals("type1", searchHit.getSourceAsMap().get("type"));
assertEquals(50, searchHit.getSourceAsMap().get("num2"));
}
@ -713,13 +713,13 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
public void testMultiSearch() throws Exception {
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
SearchRequest searchRequest1 = new SearchRequest("index1");
searchRequest1.source().sort("_id", SortOrder.ASC);
searchRequest1.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest1);
SearchRequest searchRequest2 = new SearchRequest("index2");
searchRequest2.source().sort("_id", SortOrder.ASC);
searchRequest2.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest2);
SearchRequest searchRequest3 = new SearchRequest("index3");
searchRequest3.source().sort("_id", SortOrder.ASC);
searchRequest3.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest3);
MultiSearchResponse multiSearchResponse =
@ -1211,7 +1211,8 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
assertTrue(explainResponse.hasExplanation());
assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f));
assertTrue(explainResponse.getGetResult().isExists());
assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1")));
assertEquals(2, explainResponse.getGetResult().getSource().size());
assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1"));
}
}

18
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
View File

@ -169,7 +169,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
// tag::search-source-sorting
sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2>
sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2>
// end::search-source-sorting
// tag::search-source-filtering-off
@ -1251,6 +1251,9 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
CreateIndexRequest authorsRequest = new CreateIndexRequest("authors")
.mapping(XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("user")
.field("type", "keyword")
.field("doc_values", "false")
@ -1263,6 +1266,9 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors")
.mapping(XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("user")
.field("type", "keyword")
.field("store", "true")
@ -1274,19 +1280,19 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(new IndexRequest("posts").id("1")
.source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
.source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user",
Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
bulkRequest.add(new IndexRequest("posts").id("2")
.source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
.source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user",
Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
bulkRequest.add(new IndexRequest("posts").id("3")
.source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
.source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user",
Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
bulkRequest.add(new IndexRequest("authors").id("1")
.source(XContentType.JSON, "user", "kimchy"));
.source(XContentType.JSON, "id", 1, "user", "kimchy"));
bulkRequest.add(new IndexRequest("contributors").id("1")
.source(XContentType.JSON, "user", "tanguy"));
.source(XContentType.JSON, "id", 1, "user", "tanguy"));
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

15
docs/reference/mapping/types/parent-join.asciidoc
View File

@ -16,6 +16,9 @@ PUT my_index
{
"mappings": {
"properties": {
"my_id": {
"type": "keyword"
},
"my_join_field": { <1>
"type": "join",
"relations": {
@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi
--------------------------------------------------
PUT my_index/_doc/1?refresh
{
"my_id": "1",
"text": "This is a question",
"my_join_field": {
"name": "question" <1>
@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh
PUT my_index/_doc/2?refresh
{
"my_id": "2",
"text": "This is another question",
"my_join_field": {
"name": "question"
@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation:
--------------------------------------------------
PUT my_index/_doc/1?refresh
{
"my_id": "1",
"text": "This is a question",
"my_join_field": "question" <1>
}
PUT my_index/_doc/2?refresh
{
"my_id": "2",
"text": "This is another question",
"my_join_field": "question"
}
@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents:
--------------------------------------------------
PUT my_index/_doc/3?routing=1&refresh <1>
{
"my_id": "3",
"text": "This is an answer",
"my_join_field": {
"name": "answer", <2>
@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1>
PUT my_index/_doc/4?routing=1&refresh
{
"my_id": "4",
"text": "This is another answer",
"my_join_field": {
"name": "answer",
@ -159,7 +168,7 @@ GET my_index/_search
"query": {
"match_all": {}
},
"sort": ["_id"]
"sort": ["my_id"]
}
--------------------------
// TEST[continued]
@ -183,6 +192,7 @@ Will return:
"_id": "1",
"_score": null,
"_source": {
"my_id": "1",
"text": "This is a question",
"my_join_field": "question" <1>
},
@ -196,6 +206,7 @@ Will return:
"_id": "2",
"_score": null,
"_source": {
"my_id": "2",
"text": "This is another question",
"my_join_field": "question" <2>
},
@ -210,6 +221,7 @@ Will return:
"_score": null,
"_routing": "1",
"_source": {
"my_id": "3",
"text": "This is an answer",
"my_join_field": {
"name": "answer", <3>
@ -227,6 +239,7 @@ Will return:
"_score": null,
"_routing": "1",
"_source": {
"my_id": "4",
"text": "This is another answer",
"my_join_field": {
"name": "answer",

38
modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
View File

@ -79,8 +79,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_id")
.order(SortOrder.ASC))
.addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long"))
.addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
return req;
}
@ -147,8 +146,10 @@ public class MoreExpressionTests extends ESIntegTestCase {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
client().prepareIndex("test", "doc", "1")
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test", "doc", "2")
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
assertEquals(2, rsp.getHits().getTotalHits().value);
SearchHits hits = rsp.getHits();
@ -175,8 +176,10 @@ public class MoreExpressionTests extends ESIntegTestCase {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
client().prepareIndex("test", "doc", "1")
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test", "doc", "2")
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
assertEquals(2, rsp.getHits().getTotalHits().value);
SearchHits hits = rsp.getHits();
@ -207,15 +210,18 @@ public class MoreExpressionTests extends ESIntegTestCase {
ensureGreen("test");
Map<String, Object> doc1 = new HashMap<>();
doc1.put("id", 1);
doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
doc1.put("double1", new Double[]{1.2d, 2.4d});
doc1.put("double2", 3.0d);
Map<String, Object> doc2 = new HashMap<>();
doc2.put("id", 2);
doc2.put("double0", 5.0d);
doc2.put("double1", 3.0d);
Map<String, Object> doc3 = new HashMap<>();
doc3.put("id", 3);
doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
doc3.put("double1", 4.0d);
@ -319,8 +325,8 @@ public class MoreExpressionTests extends ESIntegTestCase {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("x", 4),
client().prepareIndex("test", "doc", "2").setSource("y", 2));
client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 4),
client().prepareIndex("test", "doc","2").setSource("id", 2, "y", 2));
SearchResponse rsp = buildRequest("doc['x'] + 1").get();
ElasticsearchAssertions.assertSearchResponse(rsp);
SearchHits hits = rsp.getHits();
@ -348,9 +354,9 @@ public class MoreExpressionTests extends ESIntegTestCase {
createIndex("test");
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("x", 10),
client().prepareIndex("test", "doc", "2").setSource("x", 3),
client().prepareIndex("test", "doc", "3").setSource("x", 5));
client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 10),
client().prepareIndex("test", "doc", "2").setSource("id", 2, "x", 3),
client().prepareIndex("test", "doc", "3").setSource("id", 3, "x", 5));
// a = int, b = double, c = long
String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
@ -622,9 +628,9 @@ public class MoreExpressionTests extends ESIntegTestCase {
assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder));
ensureGreen();
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("price", 1.0, "vip", true),
client().prepareIndex("test", "doc", "2").setSource("price", 2.0, "vip", false),
client().prepareIndex("test", "doc", "3").setSource("price", 2.0, "vip", false));
client().prepareIndex("test", "doc", "1").setSource("id", 1, "price", 1.0, "vip", true),
client().prepareIndex("test", "doc", "2").setSource("id", 2, "price", 2.0, "vip", false),
client().prepareIndex("test", "doc", "3").setSource("id", 3, "price", 2.0, "vip", false));
// access .value
SearchResponse rsp = buildRequest("doc['vip'].value").get();
assertSearchResponse(rsp);
@ -653,8 +659,8 @@ public class MoreExpressionTests extends ESIntegTestCase {
createIndex("test");
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("foo", 1.0),
client().prepareIndex("test", "doc", "2").setSource("foo", 0.0));
client().prepareIndex("test", "doc", "1").setSource("id", 1, "foo", 1.0),
client().prepareIndex("test", "doc", "2").setSource("id", 2, "foo", 0.0));
SearchRequestBuilder builder = buildRequest("doc['foo'].value");
Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));

2
modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java
View File

@ -103,7 +103,7 @@ public class ChildrenIT extends AbstractParentChildTestCase {
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category").field("category").size(10000).subAggregation(
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_id", SortOrder.ASC))
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("id", SortOrder.ASC))
)
).get();
assertSearchResponse(searchResponse);

17
modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java
View File

@ -194,6 +194,9 @@ public class InnerHitsIT extends ParentChildTestCase {
public void testRandomParentChild() throws Exception {
assertAcked(prepareCreate("idx")
.addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("join_field")
.field("type", "join")
.startObject("relations")
@ -230,13 +233,13 @@ public class InnerHitsIT extends ParentChildTestCase {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None)
.innerHit(new InnerHitBuilder().setName("a")
.addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size))));
.addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size))));
boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None)
.innerHit(new InnerHitBuilder().setName("b")
.addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size))));
.addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size))));
SearchResponse searchResponse = client().prepareSearch("idx")
.setSize(numDocs)
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(boolQuery)
.get();
@ -291,7 +294,7 @@ public class InnerHitsIT extends ParentChildTestCase {
indexRandom(true, requests);
SearchResponse response = client().prepareSearch("stack")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(
boolQuery()
.must(matchQuery("body", "fail2ban"))
@ -402,7 +405,7 @@ public class InnerHitsIT extends ParentChildTestCase {
hasChildQuery("baron", matchAllQuery(), ScoreMode.None)
.innerHit(new InnerHitBuilder().setName("barons")),
ScoreMode.None).innerHit(new InnerHitBuilder()
.addSort(SortBuilders.fieldSort("_id").order(SortOrder.ASC))
.addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC))
.setName("earls")
.setSize(4))
)
@ -456,7 +459,7 @@ public class InnerHitsIT extends ParentChildTestCase {
SearchResponse response = client().prepareSearch("index")
.setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None)
.innerHit(new InnerHitBuilder()))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 2);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -473,7 +476,7 @@ public class InnerHitsIT extends ParentChildTestCase {
.innerHit(new InnerHitBuilder());
response = client().prepareSearch("index")
.setQuery(query)
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));

2
modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java
View File

@ -70,6 +70,7 @@ public abstract class ParentChildTestCase extends ESIntegTestCase {
for (int i = 0; i < fields.length; i += 2) {
source.put((String) fields[i], fields[i + 1]);
}
source.put("id", id);
return createIndexRequest(index, type, id, parentId, source);
}
@ -98,6 +99,7 @@ public abstract class ParentChildTestCase extends ESIntegTestCase {
}
joinField.put("relations", relationMap);
fields.put(joinFieldName, joinField);
fields.put("id", Collections.singletonMap("type", "keyword"));
return Collections.singletonMap("properties", fields);
}

17
modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml
View File

@ -6,46 +6,47 @@ setup:
mappings:
properties:
join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } }
id: { "type": "keyword" }
- do:
index:
index: test
id: 1
body: { "join_field": { "name": "parent" } }
body: { "id", "1", "join_field": { "name": "parent" } }
- do:
index:
index: test
id: 2
body: { "join_field": { "name": "parent" } }
body: { "id", "2", "join_field": { "name": "parent" } }
- do:
index:
index: test
id: 3
routing: 1
body: { "join_field": { "name": "child", "parent": "1" } }
body: { "id", "3", "join_field": { "name": "child", "parent": "1" } }
- do:
index:
index: test
id: 4
routing: 1
body: { "join_field": { "name": "child", "parent": "1" } }
body: { "id", "4", "join_field": { "name": "child", "parent": "1" } }
- do:
index:
index: test
id: 5
routing: 1
body: { "join_field": { "name": "child", "parent": "2" } }
body: { "id", "5", "join_field": { "name": "child", "parent": "2" } }
- do:
index:
index: test
id: 6
routing: 1
body: { "join_field": { "name": "grand_child", "parent": "5" } }
body: { "id", "6", "join_field": { "name": "grand_child", "parent": "5" } }
- do:
indices.refresh: {}
@ -55,7 +56,7 @@ setup:
- do:
search:
rest_total_hits_as_int: true
body: { sort: ["join_field", "_id"] }
body: { sort: ["join_field", "id"] }
- match: { hits.total: 6 }
- match: { hits.hits.0._index: "test" }
@ -98,7 +99,7 @@ setup:
search:
rest_total_hits_as_int: true
body:
sort: [ "_id" ]
sort: [ "id" ]
query:
parent_id:
type: child

129
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
View File

@ -44,7 +44,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
@ -69,17 +68,23 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
public void testPercolatorQuery() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
.addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
.setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "1")
.field("query", matchAllQuery()).endObject())
.get();
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "2")
.field("query", matchQuery("field1", "value")).endObject())
.get();
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject().field("query", boolQuery()
.setSource(jsonBuilder().startObject()
.field("id", "3")
.field("query", boolQuery()
.must(matchQuery("field1", "value"))
.must(matchQuery("field2", "value"))
).endObject()).get();
@ -97,7 +102,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
logger.info("percolating doc with 1 field");
response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 2);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -109,7 +114,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
logger.info("percolating doc with 2 fields");
response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 3);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -125,7 +130,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").endObject()),
BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject())
), XContentType.JSON))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 3);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -248,22 +253,26 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
public void testPercolatorGeoQueries() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator")
);
.addMapping("type", "id", "type=keyword",
"field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator"));
client().prepareIndex("test", "type", "1")
.setSource(jsonBuilder().startObject().field("query",
geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS))
.setSource(jsonBuilder().startObject()
.field("query", geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS))
.field("id", "1")
.endObject()).get();
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject().field("query",
geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6))
.setSource(jsonBuilder().startObject()
.field("query", geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6))
.field("id", "2")
.endObject()).get();
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject().field("query",
geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6))))
.setSource(jsonBuilder().startObject()
.field("query",
geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6))))
.field("id", "3")
.endObject()).get();
refresh();
@ -272,7 +281,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
.endObject());
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 3);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -282,24 +291,29 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
public void testPercolatorQueryExistingDocument() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
.addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
.setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "1")
.field("query", matchAllQuery()).endObject())
.get();
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "2")
.field("query", matchQuery("field1", "value")).endObject())
.get();
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject().field("query", boolQuery()
.setSource(jsonBuilder().startObject()
.field("id", "3")
.field("query", boolQuery()
.must(matchQuery("field1", "value"))
.must(matchQuery("field2", "value"))
).endObject()).get();
.must(matchQuery("field2", "value"))).endObject()).get();
client().prepareIndex("test", "type", "4").setSource("{}", XContentType.JSON).get();
client().prepareIndex("test", "type", "5").setSource("field1", "value").get();
client().prepareIndex("test", "type", "6").setSource("field1", "value", "field2", "value").get();
client().prepareIndex("test", "type", "4").setSource("{\"id\": \"4\"}", XContentType.JSON).get();
client().prepareIndex("test", "type", "5").setSource("id", "5", "field1", "value").get();
client().prepareIndex("test", "type", "6").setSource("id", "6", "field1", "value", "field2", "value").get();
client().admin().indices().prepareRefresh().get();
logger.info("percolating empty doc");
@ -312,7 +326,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
logger.info("percolating doc with 1 field");
response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", "test", "type", "5", null, null, null))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 2);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -321,7 +335,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
logger.info("percolating doc with 2 fields");
response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", "test", "type", "6", null, null, null))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 3);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -352,18 +366,19 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
public void testPercolatorSpecificQueries() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=text", "field2", "type=text", "query", "type=percolator")
.addMapping("type", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
.setSource(jsonBuilder().startObject().field("query", commonTermsQuery("field1", "quick brown fox")).endObject())
.get();
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2")
.setSource(jsonBuilder().startObject()
.field("id", "1")
.field("query", multiMatchQuery("quick brown fox", "field1", "field2")
.type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject())
.get();
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject().field("query",
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject()
.field("id", "2")
.field("query",
spanNearQuery(spanTermQuery("field1", "quick"), 0)
.addClause(spanTermQuery("field1", "brown"))
.addClause(spanTermQuery("field1", "fox"))
@ -372,8 +387,10 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
.get();
client().admin().indices().prepareRefresh().get();
client().prepareIndex("test", "type", "4")
.setSource(jsonBuilder().startObject().field("query",
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject()
.field("id", "3")
.field("query",
spanNotQuery(
spanNearQuery(spanTermQuery("field1", "quick"), 0)
.addClause(spanTermQuery("field1", "brown"))
@ -387,8 +404,10 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
.get();
// doesn't match
client().prepareIndex("test", "type", "5")
.setSource(jsonBuilder().startObject().field("query",
client().prepareIndex("test", "type", "4")
.setSource(jsonBuilder().startObject()
.field("id", "4")
.field("query",
spanNotQuery(
spanNearQuery(spanTermQuery("field1", "quick"), 0)
.addClause(spanTermQuery("field1", "brown"))
@ -408,7 +427,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
.endObject());
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 4);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@ -430,22 +449,32 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
fieldMapping.append(",index_options=offsets");
}
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", fieldMapping, "query", "type=percolator")
.addMapping("type", "id", "type=keyword", "field1", fieldMapping, "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
.setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "1")
.field("query", matchQuery("field1", "brown fox")).endObject())
.execute().actionGet();
client().prepareIndex("test", "type", "2")
.setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "2")
.field("query", matchQuery("field1", "lazy dog")).endObject())
.execute().actionGet();
client().prepareIndex("test", "type", "3")
.setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "3")
.field("query", termQuery("field1", "jumps")).endObject())
.execute().actionGet();
client().prepareIndex("test", "type", "4")
.setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "4")
.field("query", termQuery("field1", "dog")).endObject())
.execute().actionGet();
client().prepareIndex("test", "type", "5")
.setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject())
.setSource(jsonBuilder().startObject()
.field("id", "5")
.field("query", termQuery("field1", "fox")).endObject())
.execute().actionGet();
client().admin().indices().prepareRefresh().get();
@ -455,7 +484,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
SearchResponse searchResponse = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", document, XContentType.JSON))
.highlighter(new HighlightBuilder().field("field1"))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(searchResponse, 5);
@ -482,7 +511,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
.should(new PercolateQueryBuilder("query", document2, XContentType.JSON).setName("query2"))
)
.highlighter(new HighlightBuilder().field("field1"))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
logger.info("searchResponse={}", searchResponse);
assertHitCount(searchResponse, 5);
@ -506,7 +535,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
BytesReference.bytes(jsonBuilder().startObject().field("field1", "brown fox").endObject())
), XContentType.JSON))
.highlighter(new HighlightBuilder().field("field1"))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(searchResponse, 5);
assertThat(searchResponse.getHits().getAt(0).getFields().get("_percolator_document_slot").getValues(),
@ -546,7 +575,7 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase {
), XContentType.JSON).setName("query2"))
)
.highlighter(new HighlightBuilder().field("field1"))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
logger.info("searchResponse={}", searchResponse);
assertHitCount(searchResponse, 5);

40
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
View File

@ -66,22 +66,28 @@ public class RankEvalRequestIT extends ESIntegTestCase {
createIndex(TEST_INDEX);
ensureGreen();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("1")
.setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("2").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("3").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("4").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("5").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME).setId("6").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "1")
.setSource("id", 1, "text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "2")
.setSource("id", 2, "text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "3")
.setSource("id", 3, "text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "4")
.setSource("id", 4, "text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "5")
.setSource("id", 5, "text", "amsterdam", "population", 851573)
.get();
client().prepareIndex(TEST_INDEX, MapperService.SINGLE_MAPPING_NAME, "6")
.setSource("id", 6, "text", "amsterdam", "population", 851573)
.get();
// add another index for testing closed indices etc...
client().prepareIndex("test2", MapperService.SINGLE_MAPPING_NAME).setId("7").setSource("text", "amsterdam", "population", 851573)
.get();
client().prepareIndex("test2", MapperService.SINGLE_MAPPING_NAME, "7")
.setSource("id", 7, "text", "amsterdam", "population", 851573)
.get();
refresh();
// set up an alias that can also be used in tests
@ -97,7 +103,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
List<RatedRequest> specifications = new ArrayList<>();
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
testQuery.sort("id");
RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
createRelevant("2", "3", "4", "5"), testQuery);
amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
@ -174,7 +180,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
public void testDCGRequest() {
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
testQuery.sort("id");
List<RatedRequest> specifications = new ArrayList<>();
List<RatedDocument> ratedDocs = Arrays.asList(
@ -208,7 +214,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
public void testMRRRequest() {
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
testQuery.sort("id");
List<RatedRequest> specifications = new ArrayList<>();
specifications.add(new RatedRequest("amsterdam_query", createRelevant("5"), testQuery));

89
plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
View File

@ -64,6 +64,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "tr")
@ -75,8 +78,10 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
// both values should collate to same value
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1")
.setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2")
.setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
);
// searching for either of the terms should return both results since they collate to the same value
@ -87,7 +92,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.fetchSource(false)
.query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
.sort("collate")
.sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
.sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
);
SearchResponse response = client().search(request).actionGet();
@ -104,6 +109,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "en")
@ -114,9 +122,10 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
// everything should be indexed fine, no exceptions
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":[\"" + equivalent[0] + "\", \""
+ equivalent[1] + "\"]}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1")
.setSource("{\"id\":\"1\", \"collate\":[\"" + equivalent[0] + "\", \"" + equivalent[1] + "\"]}", XContentType.JSON),
client().prepareIndex(index, type, "2")
.setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON)
);
// using sort mode = max, values B and C will be used for the sort
@ -128,7 +137,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.query(QueryBuilders.termQuery("collate", "a"))
// if mode max we use c and b as sort values, if max we use "a" for both
.sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MAX).order(SortOrder.DESC))
.sort("_id", SortOrder.DESC) // will be ignored
.sort("id", SortOrder.DESC) // will be ignored
);
SearchResponse response = client().search(request).actionGet();
@ -145,7 +154,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.query(QueryBuilders.termQuery("collate", "a"))
// if mode max we use c and b as sort values, if max we use "a" for both
.sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MIN).order(SortOrder.DESC))
.sort("_id", SortOrder.DESC) // will NOT be ignored and will determine order
.sort("id", SortOrder.DESC) // will NOT be ignored and will determine order
);
response = client().search(request).actionGet();
@ -165,6 +174,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "tr")
@ -176,8 +188,10 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1")
.setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2")
.setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
);
// searching for either of the terms should return both results since they collate to the same value
@ -188,7 +202,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.fetchSource(false)
.query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
.sort("collate")
.sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
.sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
);
SearchResponse response = client().search(request).actionGet();
@ -208,6 +222,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "en")
@ -219,8 +236,10 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1")
.setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2")
.setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
);
SearchRequest request = new SearchRequest()
@ -230,7 +249,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.fetchSource(false)
.query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
.sort("collate")
.sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
.sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
);
SearchResponse response = client().search(request).actionGet();
@ -251,6 +270,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "en")
@ -262,8 +284,8 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
);
SearchRequest request = new SearchRequest()
@ -273,7 +295,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.fetchSource(false)
.query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
.sort("collate")
.sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
.sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
);
SearchResponse response = client().search(request).actionGet();
@ -292,6 +314,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "en")
@ -305,9 +330,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"foo bar\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"foobar\"}", XContentType.JSON),
client().prepareIndex(index, type, "3").setSource("{\"collate\":\"foo-bar\"}", XContentType.JSON)
client().prepareIndex(index, type, "1").setSource("{\"id\":\"1\",\"collate\":\"foo bar\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"id\":\"2\",\"collate\":\"foobar\"}", XContentType.JSON),
client().prepareIndex(index, type, "3").setSource("{\"id\":\"3\",\"collate\":\"foo-bar\"}", XContentType.JSON)
);
SearchRequest request = new SearchRequest()
@ -316,7 +341,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.source(new SearchSourceBuilder()
.fetchSource(false)
.sort("collate", SortOrder.ASC)
.sort("_id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value
.sort("id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value
);
SearchResponse response = client().search(request).actionGet();
@ -374,6 +399,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("language", "en")
@ -386,10 +414,10 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"résumé\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON),
client().prepareIndex(index, type, "3").setSource("{\"collate\":\"resume\"}", XContentType.JSON),
client().prepareIndex(index, type, "4").setSource("{\"collate\":\"Résumé\"}", XContentType.JSON)
client().prepareIndex(index, type, "1").setSource("{\"id\":\"1\",\"collate\":\"résumé\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"id\":\"2\",\"collate\":\"Resume\"}", XContentType.JSON),
client().prepareIndex(index, type, "3").setSource("{\"id\":\"3\",\"collate\":\"resume\"}", XContentType.JSON),
client().prepareIndex(index, type, "4").setSource("{\"id\":\"4\",\"collate\":\"Résumé\"}", XContentType.JSON)
);
SearchRequest request = new SearchRequest()
@ -398,7 +426,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.source(new SearchSourceBuilder()
.fetchSource(false)
.sort("collate", SortOrder.ASC)
.sort("_id", SortOrder.DESC)
.sort("id", SortOrder.DESC)
);
SearchResponse response = client().search(request).actionGet();
@ -471,6 +499,9 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
XContentBuilder builder = jsonBuilder()
.startObject().startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("collate")
.field("type", "icu_collation_keyword")
.field("rules", tailoredRules)
@ -481,8 +512,8 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
indexRandom(true,
client().prepareIndex(index, type, "1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
client().prepareIndex(index, type, "1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON),
client().prepareIndex(index, type, "2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON)
);
SearchRequest request = new SearchRequest()
@ -492,7 +523,7 @@ public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
.fetchSource(false)
.query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
.sort("collate", SortOrder.ASC)
.sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
.sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value
);
SearchResponse response = client().search(request).actionGet();

6
qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
View File

@ -165,7 +165,7 @@ public class CCSDuelIT extends ESRestTestCase {
//this index with a single document is used to test partial failures
IndexRequest indexRequest = new IndexRequest(INDEX_NAME + "_err");
indexRequest.id("id");
indexRequest.source("creationDate", "err");
indexRequest.source("id", "id", "creationDate", "err");
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
assertEquals(201, indexResponse.status().getStatus());
@ -178,6 +178,7 @@ public class CCSDuelIT extends ESRestTestCase {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
createIndexRequest.settings(Settings.builder().put("index.number_of_shards", numShards).put("index.number_of_replicas", 0));
createIndexRequest.mapping("{\"properties\":{" +
"\"id\":{\"type\":\"keyword\"}," +
"\"suggest\":{\"type\":\"completion\"}," +
"\"join\":{\"type\":\"join\", \"relations\": {\"question\":\"answer\"}}}}", XContentType.JSON);
CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
@ -237,6 +238,7 @@ public class CCSDuelIT extends ESRestTestCase {
joinField.put("parent", questionId);
}
indexRequest.source(XContentType.JSON,
"id", id,
"type", type,
"votes", randomIntBetween(0, 30),
"questionId", questionId,
@ -614,7 +616,7 @@ public class CCSDuelIT extends ESRestTestCase {
topHits.from(10);
topHits.size(10);
topHits.sort("creationDate", SortOrder.DESC);
topHits.sort("_id", SortOrder.ASC);
topHits.sort("id", SortOrder.ASC);
TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING);
tags.field("tags.keyword");
tags.size(10);

2
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml
View File

@ -42,7 +42,7 @@
rest_total_hits_as_int: true
index: queries
body:
sort: _id
sort: id
query:
percolate:
field: query

9
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
View File

@ -66,6 +66,8 @@
body:
mappings:
properties:
id:
type: keyword
query:
type: percolator
field1:
@ -80,6 +82,7 @@
index: queries
id: q1
body:
id: q1
query:
term:
field1: value
@ -89,6 +92,7 @@
index: queries
id: q2
body:
id: q2
query:
bool:
must:
@ -102,6 +106,7 @@
index: queries
id: q3
body:
id: q3
query:
bool:
minimum_should_match: 2
@ -133,7 +138,7 @@
rest_total_hits_as_int: true
index: queries
body:
sort: _id
sort: id
query:
percolate:
field: query
@ -149,7 +154,7 @@
rest_total_hits_as_int: true
index: queries
body:
sort: _id
sort: id
query:
percolate:
field: query

3
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
View File

@ -30,6 +30,7 @@
id: q4
refresh: true
body:
id: q4
query:
bool:
minimum_should_match: 2
@ -57,7 +58,7 @@
rest_total_hits_as_int: true
index: queries
body:
sort: _id
sort: id
query:
percolate:
field: query

26
rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
View File

@ -6,19 +6,19 @@ setup:
index:
index: test
id: 1
body: { foo: bar, age: 18 }
body: { id: 1, foo: bar, age: 18 }
- do:
index:
index: test
id: 42
body: { foo: bar, age: 18 }
body: { id: 42, foo: bar, age: 18 }
- do:
index:
index: test
id: 172
body: { foo: bar, age: 24 }
body: { id: 172, foo: bar, age: 24 }
- do:
indices.refresh:
@ -36,14 +36,14 @@ setup:
query:
match:
foo: bar
sort: [{ age: desc }, { _id: desc }]
sort: [{ age: desc }, { id: desc }]
- match: {hits.total: 3 }
- length: {hits.hits: 1 }
- match: {hits.hits.0._index: test }
- match: {hits.hits.0._type: _doc }
- match: {hits.hits.0._id: "172" }
- match: {hits.hits.0.sort: [24, "172"] }
- match: {hits.hits.0.sort: [24, 172] }
- do:
search:
@ -54,15 +54,15 @@ setup:
query:
match:
foo: bar
sort: [{ age: desc }, { _id: desc }]
search_after: [24, "172"]
sort: [{ age: desc }, { id: desc }]
search_after: [24, 172]
- match: {hits.total: 3 }
- length: {hits.hits: 1 }
- match: {hits.hits.0._index: test }
- match: {hits.hits.0._type: _doc }
- match: {hits.hits.0._id: "42" }
- match: {hits.hits.0.sort: [18, "42"] }
- match: {hits.hits.0.sort: [18, 42] }
- do:
search:
@ -73,15 +73,15 @@ setup:
query:
match:
foo: bar
sort: [ { age: desc }, { _id: desc } ]
search_after: [18, "42"]
sort: [ { age: desc }, { id: desc } ]
search_after: [18, 42]
- match: {hits.total: 3}
- length: {hits.hits: 1 }
- match: {hits.hits.0._index: test }
- match: {hits.hits.0._type: _doc }
- match: {hits.hits.0._id: "1" }
- match: {hits.hits.0.sort: [18, "1"] }
- match: {hits.hits.0.sort: [18, 1] }
- do:
search:
@ -92,8 +92,8 @@ setup:
query:
match:
foo: bar
sort: [{ age: desc }, { _id: desc } ]
search_after: [18, "1"]
sort: [{ age: desc }, { id: desc } ]
search_after: [18, 1]
- match: {hits.total: 3}
- length: {hits.hits: 0 }

2
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View File

@ -183,7 +183,7 @@ public class MetaDataIndexUpgradeService {
try (IndexAnalyzers fakeIndexAnalzyers =
new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) {
MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService,
mapperRegistry, () -> null);
mapperRegistry, () -> null, () -> false);
mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY);
}
} catch (Exception ex) {

1
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -210,6 +210,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING,
IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING,
IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING,
IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING,
MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING,
MetaData.SETTING_READ_ONLY_SETTING,
MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING,

8
server/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -74,6 +74,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiFunction;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
@ -394,7 +395,8 @@ public final class IndexModule {
IndicesQueryCache indicesQueryCache,
MapperRegistry mapperRegistry,
IndicesFieldDataCache indicesFieldDataCache,
NamedWriteableRegistry namedWriteableRegistry)
NamedWriteableRegistry namedWriteableRegistry,
BooleanSupplier idFieldDataEnabled)
throws IOException {
final IndexEventListener eventListener = freeze();
Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory =
@ -422,7 +424,7 @@ public final class IndexModule {
new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers,
engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache,
directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners,
indexOperationListeners, namedWriteableRegistry);
indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled);
success = true;
return indexService;
} finally {
@ -469,7 +471,7 @@ public final class IndexModule {
ScriptService scriptService) throws IOException {
return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry,
new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry,
() -> { throw new UnsupportedOperationException("no index query shard context available"); });
() -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false);
}
/**

6
server/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -92,6 +92,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.LongSupplier;
@ -162,7 +163,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
IndicesFieldDataCache indicesFieldDataCache,
List<SearchOperationListener> searchOperationListeners,
List<IndexingOperationListener> indexingOperationListeners,
NamedWriteableRegistry namedWriteableRegistry) {
NamedWriteableRegistry namedWriteableRegistry,
BooleanSupplier idFieldDataEnabled) {
super(indexSettings);
this.indexSettings = indexSettings;
this.xContentRegistry = xContentRegistry;
@ -173,7 +175,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
assert indexAnalyzers != null;
this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
// we parse all percolator queries as they would be parsed on shard 0
() -> newQueryShardContext(0, null, System::currentTimeMillis, null));
() -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled);
this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
if (indexSettings.getIndexSortConfig().hasIndexSort()) {
// we delay the actual creation of the sort order for this index because the mapping has not been merged yet.

14
server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.mapper;
import org.apache.logging.log4j.LogManager;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
@ -28,6 +29,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
@ -41,6 +43,7 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
@ -55,6 +58,11 @@ import java.util.Map;
* queries.
*/
public class IdFieldMapper extends MetadataFieldMapper {
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(IdFieldMapper.class));
static final String ID_FIELD_DATA_DEPRECATION_MESSAGE =
"Loading the fielddata on the _id field is deprecated and will be removed in future versions. "
+ "If you require sorting or aggregating on this field you should also include the id in the "
+ "body of your documents, and map this field as a keyword field that has [doc_values] enabled";
public static final String NAME = "_id";
@ -158,6 +166,12 @@ public class IdFieldMapper extends MetadataFieldMapper {
@Override
public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService) {
if (mapperService.isIdFieldDataEnabled() == false) {
throw new IllegalArgumentException("Fielddata access on the _id field is disallowed, "
+ "you can re-enable it by updating the dynamic cluster setting: "
+ IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey());
}
deprecationLogger.deprecatedAndMaybeLog("id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE);
final IndexFieldData<?> fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache,
breakerService, mapperService);
return new IndexFieldData<AtomicFieldData>() {

13
server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -72,6 +72,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
@ -151,9 +152,11 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
final MapperRegistry mapperRegistry;
private final BooleanSupplier idFieldDataEnabled;
public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry,
SimilarityService similarityService, MapperRegistry mapperRegistry,
Supplier<QueryShardContext> queryShardContextSupplier) {
Supplier<QueryShardContext> queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) {
super(indexSettings);
this.indexAnalyzers = indexAnalyzers;
this.fieldTypes = new FieldTypeLookup();
@ -163,6 +166,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer());
this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
this.mapperRegistry = mapperRegistry;
this.idFieldDataEnabled = idFieldDataEnabled;
if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings()) &&
indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0)) {
@ -817,6 +821,13 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
return this.searchQuoteAnalyzer;
}
/**
* Returns <code>true</code> if fielddata is enabled for the {@link IdFieldMapper} field, <code>false</code> otherwise.
*/
public boolean isIdFieldDataEnabled() {
return idFieldDataEnabled.getAsBoolean();
}
@Override
public void close() throws IOException {
indexAnalyzers.close();

21
server/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -92,6 +92,7 @@ import org.elasticsearch.index.engine.NoOpEngine;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.mapper.IdFieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.query.QueryBuilder;
@ -169,6 +170,9 @@ public class IndicesService extends AbstractLifecycleComponent
public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING =
Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope);
public static final Setting<Boolean> INDICES_ID_FIELD_DATA_ENABLED_SETTING =
Setting.boolSetting("indices.id_field_data.enabled", true, Property.Dynamic, Property.NodeScope);
/**
* The node's settings.
@ -204,6 +208,7 @@ public class IndicesService extends AbstractLifecycleComponent
private final Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories;
final AbstractRefCounted indicesRefCount; // pkg-private for testing
private final CountDownLatch closeLatch = new CountDownLatch(1);
private volatile boolean idFieldDataEnabled;
@Override
protected void doStart() {
@ -239,6 +244,8 @@ public class IndicesService extends AbstractLifecycleComponent
this.scriptService = scriptService;
this.clusterService = clusterService;
this.client = client;
this.idFieldDataEnabled = INDICES_ID_FIELD_DATA_ENABLED_SETTING.get(clusterService.getSettings());
clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_ID_FIELD_DATA_ENABLED_SETTING, this::setIdFieldDataEnabled);
this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
@Override
public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
@ -564,7 +571,8 @@ public class IndicesService extends AbstractLifecycleComponent
indicesQueryCache,
mapperRegistry,
indicesFieldDataCache,
namedWriteableRegistry
namedWriteableRegistry,
this::isIdFieldDataEnabled
);
}
@ -1452,6 +1460,17 @@ public class IndicesService extends AbstractLifecycleComponent
return mapperRegistry.isMetaDataField(indexCreatedVersion, field);
}
/**
* Returns <code>true</code> if fielddata is enabled for the {@link IdFieldMapper} field, <code>false</code> otherwise.
*/
public boolean isIdFieldDataEnabled() {
return idFieldDataEnabled;
}
private void setIdFieldDataEnabled(boolean value) {
this.idFieldDataEnabled = value;
}
/**
* Checks to see if an operation can be performed without taking the cluster over the cluster-wide shard limit. Adds a deprecation
* warning or returns an error message as appropriate

2
server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -160,7 +160,7 @@ public class IndexModuleTests extends ESTestCase {
private IndexService newIndexService(IndexModule module) throws IOException {
return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays,
threadPool, scriptService, clusterService, null, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener), writableRegistry());
new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false);
}
public void testWrapperIsBound() throws IOException {

2
server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
View File

@ -94,7 +94,7 @@ public class CodecTests extends ESTestCase {
IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers;
MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry,
() -> null);
() -> null, () -> false);
return new CodecService(service, LogManager.getLogger("test"));
}

32
server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
View File

@ -28,7 +28,9 @@ import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
@ -37,6 +39,9 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE;
import static org.hamcrest.Matchers.containsString;
public class IdFieldMapperTests extends ESSingleNodeTestCase {
@Override
@ -71,4 +76,31 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase {
assertEquals(Uid.encodeId("id"), fields[0].binaryValue());
}
public void testEnableFieldData() throws IOException {
IndexService service = createIndex("test", Settings.EMPTY);
MapperService mapperService = service.mapperService();
mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE);
IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) service.mapperService().fullName("_id");
ft.fielddataBuilder("test").build(mapperService.getIndexSettings(),
ft, null, null, mapperService);
assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE);
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), false))
.get();
try {
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
() -> ft.fielddataBuilder("test").build(mapperService.getIndexSettings(),
ft, null, null, mapperService));
assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()));
} finally {
// unset cluster setting
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()))
.get();
}
}
}

5
server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
View File

@ -119,6 +119,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; i++) {
docs[i] = client().prepareIndex("test", "type1", String.valueOf(i)).setSource(
"id", String.valueOf(i),
"field1", English.intToEnglish(i),
"field2", i
);
@ -136,14 +137,14 @@ public class QueryProfilerIT extends ESIntegTestCase {
SearchRequestBuilder vanilla = client().prepareSearch("test")
.setQuery(q)
.setProfile(false)
.addSort("_id", SortOrder.ASC)
.addSort("id.keyword", SortOrder.ASC)
.setSearchType(SearchType.QUERY_THEN_FETCH)
.setRequestCache(false);
SearchRequestBuilder profile = client().prepareSearch("test")
.setQuery(q)
.setProfile(true)
.addSort("_id", SortOrder.ASC)
.addSort("id.keyword", SortOrder.ASC)
.setSearchType(SearchType.QUERY_THEN_FETCH)
.setRequestCache(false);

35
server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
View File

@ -99,6 +99,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
int numDocs = scaledRandomIntBetween(50, 100);
List<IndexRequestBuilder> builders = new ArrayList<>();
builders.add(client().prepareIndex("test", "test", "theone").setSource(
"id", "theone",
"full_name", "Captain America",
"first_name", "Captain",
"last_name", "America",
@ -106,6 +107,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
"skill", 15,
"int-field", 25));
builders.add(client().prepareIndex("test", "test", "theother").setSource(
"id", "theother",
"full_name", "marvel hero",
"first_name", "marvel",
"last_name", "hero",
@ -113,6 +115,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
"skill", 5));
builders.add(client().prepareIndex("test", "test", "ultimate1").setSource(
"id", "ultimate1",
"full_name", "Alpha the Ultimate Mutant",
"first_name", "Alpha the",
"last_name", "Ultimate Mutant",
@ -126,6 +129,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
"skill", 3));
builders.add(client().prepareIndex("test", "test", "anotherhero").setSource(
"id", "anotherhero",
"full_name", "ultimate",
"first_name", "wolferine",
"last_name", "",
@ -133,6 +137,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
"skill", 1));
builders.add(client().prepareIndex("test", "test", "nowHero").setSource(
"id", "nowHero",
"full_name", "now sort of",
"first_name", "now",
"last_name", "",
@ -149,6 +154,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
String first = RandomPicks.randomFrom(random(), firstNames);
String last = randomPickExcept(lastNames, first);
builders.add(client().prepareIndex("test", "test", "" + i).setSource(
"id", i,
"full_name", first + " " + last,
"first_name", first,
"last_name", last,
@ -161,6 +167,9 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
private XContentBuilder createMapping() throws IOException {
return XContentFactory.jsonBuilder().startObject().startObject("test")
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("full_name")
.field("type", "text")
.field("copy_to", "full_name_phrase")
@ -276,17 +285,17 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
}
MultiMatchQueryBuilder multiMatchQueryBuilder = randomizeType(multiMatchQuery(builder.toString(), field));
SearchResponse multiMatchResp = client().prepareSearch("test")
// _id sort field is a tie, in case hits have the same score,
// id sort field is a tie, in case hits have the same score,
// the hits will be sorted the same consistently
.addSort("_score", SortOrder.DESC)
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(multiMatchQueryBuilder).get();
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(field, builder.toString());
SearchResponse matchResp = client().prepareSearch("test")
// _id tie sort
// id tie sort
.addSort("_score", SortOrder.DESC)
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(matchQueryBuilder).get();
assertThat("field: " + field + " query: " + builder.toString(), multiMatchResp.getHits().getTotalHits().value,
equalTo(matchResp.getHits().getTotalHits().value));
@ -374,12 +383,12 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category") :
multiMatchQuery("marvel hero captain america", "*_name", randomBoolean() ? "category" : "categ*");
SearchResponse left = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(randomizeType(multiMatchQueryBuilder
.operator(Operator.OR).type(type))).get();
SearchResponse right = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(disMaxQuery().
add(matchQuery("full_name", "marvel hero captain america"))
.add(matchQuery("first_name", "marvel hero captain america"))
@ -397,12 +406,12 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") :
multiMatchQuery("captain america", "*_name", randomBoolean() ? "category" : "categ*");
SearchResponse left = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(randomizeType(multiMatchQueryBuilder
.operator(op).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch).type(type))).get();
SearchResponse right = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(boolQuery().minimumShouldMatch(minShouldMatch)
.should(randomBoolean() ? termQuery("full_name", "captain america") :
matchQuery("full_name", "captain america").operator(op))
@ -416,12 +425,12 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
{
String minShouldMatch = randomBoolean() ? null : "" + between(0, 1);
SearchResponse left = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(randomizeType(multiMatchQuery("capta", "full_name", "first_name", "last_name", "category")
.type(MatchQuery.Type.PHRASE_PREFIX).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get();
SearchResponse right = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(boolQuery().minimumShouldMatch(minShouldMatch)
.should(matchPhrasePrefixQuery("full_name", "capta"))
.should(matchPhrasePrefixQuery("first_name", "capta"))
@ -435,17 +444,17 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
SearchResponse left;
if (randomBoolean()) {
left = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category")
.type(MatchQuery.Type.PHRASE).minimumShouldMatch(minShouldMatch))).get();
} else {
left = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category")
.type(MatchQuery.Type.PHRASE).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get();
}
SearchResponse right = client().prepareSearch("test").setSize(numDocs)
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id"))
.addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id"))
.setQuery(boolQuery().minimumShouldMatch(minShouldMatch)
.should(matchPhraseQuery("full_name", "captain america"))
.should(matchPhraseQuery("first_name", "captain america"))

31
server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.WrapperQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.index.search.MatchQuery;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.TermsLookup;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
@ -1857,18 +1858,28 @@ public class SearchQueryIT extends ESIntegTestCase {
.setRouting("custom")
.setSource("field", "value");
indexRandom(true, false, indexRequest);
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true))
.get();
try {
SearchResponse searchResponse = client().prepareSearch()
.setQuery(termQuery("routing-alias", "custom"))
.addDocValueField("id-alias")
.get();
assertHitCount(searchResponse, 1L);
SearchResponse searchResponse = client().prepareSearch()
.setQuery(termQuery("routing-alias", "custom"))
.addDocValueField("id-alias")
.get();
assertHitCount(searchResponse, 1L);
SearchHit hit = searchResponse.getHits().getAt(0);
assertEquals(2, hit.getFields().size());
assertTrue(hit.getFields().containsKey("id-alias"));
SearchHit hit = searchResponse.getHits().getAt(0);
assertEquals(2, hit.getFields().size());
assertTrue(hit.getFields().containsKey("id-alias"));
DocumentField field = hit.getFields().get("id-alias");
assertThat(field.getValue().toString(), equalTo("1"));
} finally {
// unset cluster setting
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()))
.get();
}
DocumentField field = hit.getFields().get("id-alias");
assertThat(field.getValue().toString(), equalTo("1"));
}
}

50
server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.script.MockScriptPlugin;
@ -1371,31 +1372,42 @@ public class FieldSortIT extends ESIntegTestCase {
}
public void testSortMetaField() throws Exception {
createIndex("test");
ensureGreen();
final int numDocs = randomIntBetween(10, 20);
IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; ++i) {
indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i))
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true))
.get();
try {
createIndex("test");
ensureGreen();
final int numDocs = randomIntBetween(10, 20);
IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; ++i) {
indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i))
.setSource();
}
indexRandom(true, indexReqs);
}
indexRandom(true, indexReqs);
SortOrder order = randomFrom(SortOrder.values());
SearchResponse searchResponse = client().prepareSearch()
SortOrder order = randomFrom(SortOrder.values());
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.setSize(randomIntBetween(1, numDocs + 5))
.addSort("_id", order)
.get();
assertNoFailures(searchResponse);
SearchHit[] hits = searchResponse.getHits().getHits();
BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM;
for (int i = 0; i < hits.length; ++i) {
String idString = hits[i].getId();
final BytesRef id = new BytesRef(idString);
assertEquals(idString, hits[i].getSortValues()[0]);
assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id));
previous = id;
assertNoFailures(searchResponse);
SearchHit[] hits = searchResponse.getHits().getHits();
BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM;
for (int i = 0; i < hits.length; ++i) {
String idString = hits[i].getId();
final BytesRef id = new BytesRef(idString);
assertEquals(idString, hits[i].getSortValues()[0]);
assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id));
previous = id;
}
// assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE);
} finally {
// unset cluster setting
client().admin().cluster().prepareUpdateSettings()
.setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()))
.get();
}
}

2
test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View File

@ -66,6 +66,6 @@ public class MapperTestUtils {
xContentRegistry,
similarityService,
mapperRegistry,
() -> null);
() -> null, () -> false);
}
}

2
test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
View File

@ -68,7 +68,7 @@ public class TranslogHandler implements Engine.TranslogRecoveryRunner {
SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap());
MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry();
mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
() -> null);
() -> null, () -> false);
}
private DocumentMapperForType docMapper(String type) {

2
test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
View File

@ -357,7 +357,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap());
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
() -> createShardContext(null));
() -> createShardContext(null), () -> false);
IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
});
indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,

43
x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
View File

@ -76,7 +76,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
" - names: '*'\n" +
" privileges: [ ALL ]\n" +
" field_security:\n" +
" grant: [ field1 ]\n" +
" grant: [ field1, id ]\n" +
" query: '{\"term\" : {\"field1\" : \"value1\"}}'\n" +
"role3:\n" +
" cluster: [ all ]\n" +
@ -84,7 +84,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
" - names: '*'\n" +
" privileges: [ ALL ]\n" +
" field_security:\n" +
" grant: [ field2 ]\n" +
" grant: [ field2, id ]\n" +
" query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" +
"role4:\n" +
" cluster: [ all ]\n" +
@ -92,7 +92,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
" - names: '*'\n" +
" privileges: [ ALL ]\n" +
" field_security:\n" +
" grant: [ field1 ]\n" +
" grant: [ field1, id ]\n" +
" query: '{\"term\" : {\"field2\" : \"value2\"}}'\n";
}
@ -106,12 +106,12 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
public void testSimpleQuery() {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type1", "field1", "type=text", "field2", "type=text")
.addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text")
);
client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
client().prepareIndex("test", "type1", "1").setSource("id", "1", "field1", "value1")
.setRefreshPolicy(IMMEDIATE)
.get();
client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
client().prepareIndex("test", "type1", "2").setSource("id", "2", "field2", "value2")
.setRefreshPolicy(IMMEDIATE)
.get();
@ -121,20 +121,22 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
.get();
assertHitCount(response, 1);
assertSearchHits(response, "1");
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1").toString(), equalTo("value1"));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("1"));
response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
.prepareSearch("test")
.get();
assertHitCount(response, 1);
assertSearchHits(response, "2");
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2").toString(), equalTo("value2"));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("2"));
response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
.prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 2);
assertSearchHits(response, "1", "2");
@ -172,12 +174,12 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
public void testQueryCache() {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
.addMapping("type1", "field1", "type=text", "field2", "type=text")
.addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text")
);
client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
client().prepareIndex("test", "type1", "1").setSource("id", "1", "field1", "value1")
.setRefreshPolicy(IMMEDIATE)
.get();
client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
client().prepareIndex("test", "type1", "2").setSource("id", "2", "field2", "value2")
.setRefreshPolicy(IMMEDIATE)
.get();
@ -190,15 +192,17 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
.get();
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1"));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1"));
response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
.prepareSearch("test")
.get();
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2"), equalTo("value2"));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2"));
// this is a bit weird the document level permission (all docs with field2:value2) don't match with the field level
// permissions (field1),
@ -208,21 +212,24 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
.get();
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(0));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2"));
// user4 has all roles
response = client().filterWithHeader(
Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
.prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(response, 2);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1"));
assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1"));
assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(1).getSourceAsMap().get("field2"), equalTo("value2"));
assertThat(response.getHits().getAt(1).getSourceAsMap().get("id"), equalTo("2"));
}
}

8
x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
View File

@ -608,6 +608,9 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
public void testParentChild() throws Exception {
XContentBuilder mapping = jsonBuilder().startObject()
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("join_field")
.field("type", "join")
.startObject("relations")
@ -634,15 +637,18 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
Map<String, Object> source = new HashMap<>();
source.put("field2", "value2");
source.put("id", "c1");
Map<String, Object> joinField = new HashMap<>();
joinField.put("name", "child");
joinField.put("parent", "p1");
source.put("join_field", joinField);
client().prepareIndex("test", "doc", "c1").setSource(source).setRouting("p1").get();
source.put("id", "c2");
client().prepareIndex("test", "doc", "c2").setSource(source).setRouting("p1").get();
source = new HashMap<>();
source.put("field3", "value3");
source.put("join_field", joinField);
source.put("id", "c3");
client().prepareIndex("test", "doc", "c3").setSource(source).setRouting("p1").get();
refresh();
verifyParentChild();
@ -657,7 +663,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
searchResponse = client().prepareSearch("test")
.setQuery(hasParentQuery("parent", matchAllQuery(), false))
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.get();
assertHitCount(searchResponse, 3L);
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("c1"));

23
x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
View File

@ -64,6 +64,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
allowedFields = new HashSet<>();
disAllowedFields = new HashSet<>();
int numFields = scaledRandomIntBetween(5, 50);
allowedFields.add("id");
for (int i = 0; i < numFields; i++) {
String field = "field" + i;
if (i % 2 == 0) {
@ -100,21 +101,21 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
" privileges:\n" +
" - all\n" +
" field_security:\n" +
" grant: [ field1 ]\n" +
" grant: [ id, field1 ]\n" +
"role4:\n" +
" cluster: [ all ]\n" +
" indices:\n" +
" - names: test\n" +
" privileges: [ ALL ]\n" +
" field_security:\n" +
" grant: [ field2 ]\n" +
" grant: [ id, field2 ]\n" +
"role5:\n" +
" cluster: [ all ]\n" +
" indices:\n" +
" - names: test\n" +
" privileges: [ ALL ]\n" +
" field_security:\n" +
" grant: [ field3 ]\n";
" grant: [ id, field3 ]\n";
}
@Override
@ -166,7 +167,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
public void testDuel() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type1", "field1", "type=text", "field2", "type=text", "field3", "type=text")
.addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "field3", "type=text")
);
int numDocs = scaledRandomIntBetween(32, 128);
@ -174,14 +175,14 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
for (int i = 1; i <= numDocs; i++) {
String field = randomFrom("field1", "field2", "field3");
String value = "value";
requests.add(client().prepareIndex("test", "type1", value).setSource(field, value));
requests.add(client().prepareIndex("test", "type1", value).setSource("id", Integer.toString(i), field, value));
}
indexRandom(true, requests);
SearchResponse actual = client()
.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
.prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field1", "value"))
.should(QueryBuilders.termQuery("field2", "value"))
@ -189,7 +190,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
)
.get();
SearchResponse expected = client().prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field1", "value"))
)
@ -202,7 +203,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
.prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field1", "value"))
.should(QueryBuilders.termQuery("field2", "value"))
@ -210,7 +211,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
)
.get();
expected = client().prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field2", "value"))
)
@ -223,7 +224,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
.prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field1", "value"))
.should(QueryBuilders.termQuery("field2", "value"))
@ -231,7 +232,7 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase {
)
.get();
expected = client().prepareSearch("test")
.addSort("_id", SortOrder.ASC)
.addSort("id", SortOrder.ASC)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("field3", "value"))
)