From 3ed0da5a586e8a3d12016aa517ab46032353d179 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 25 Aug 2016 11:42:27 +0200 Subject: [PATCH] GET operations should not extract fields from `_source`. #20158 This makes GET operations more consistent with `_search` operations which expect `(stored_)fields` to work on stored fields and source filtering to work on the `_source` field. This is now possible thanks to the fact that GET operations do not read from the translog anymore (#20102) and also allows to get rid of `FieldMapper#isGenerated`. The `_termvectors` API (and thus more_like_this too) was relying on the fact that GET operations would extract fields from either stored fields or the source so the logic to do this that used to exist in `ShardGetService` has been moved to `TermVectorsService`. It would be nice that term vectors do not rely on this, but this does not seem to be a low hanging fruit. --- .../index/get/ShardGetService.java | 36 ---------- .../index/mapper/AllFieldMapper.java | 4 -- .../index/mapper/FieldMapper.java | 10 --- .../index/mapper/FieldNamesFieldMapper.java | 4 -- .../mapper/LegacyTokenCountFieldMapper.java | 5 -- .../index/mapper/TokenCountFieldMapper.java | 5 -- .../index/termvectors/TermVectorsService.java | 41 ++++++++--- .../action/bulk/BulkWithUpdatesIT.java | 26 +++---- .../AliasedIndexDocumentActionsIT.java | 4 +- .../document/DocumentActionsIT.java | 3 +- .../explain/ExplainActionIT.java | 4 +- .../org/elasticsearch/get/GetActionIT.java | 8 ++- .../index/IndexWithShadowReplicasIT.java | 72 +++++++++---------- .../mapper/murmur3/Murmur3FieldMapper.java | 5 -- .../rest-api-spec/test/get/20_fields.yaml | 14 ++++ .../test/get/70_source_filtering.yaml | 12 ++++ .../rest-api-spec/test/mget/20_fields.yaml | 14 ++++ 17 files changed, 133 insertions(+), 134 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 67237864595..fb3fb5aa56d 100644 --- a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -41,28 +41,19 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParentFieldMapper; -import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.mapper.TTLFieldMapper; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.translog.Translog; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.ParentFieldSubFetchPhase; -import org.elasticsearch.search.lookup.LeafSearchLookup; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -218,41 +209,14 @@ public final class ShardGetService extends AbstractIndexShardComponent { fields.put(ParentFieldMapper.NAME, new GetField(ParentFieldMapper.NAME, Collections.singletonList(parentId))); } - // now, go and do the script thingy if needed - if (gFields != null && gFields.length > 0) { - SearchLookup searchLookup = null; for (String field : gFields) { - Object value = null; FieldMapper fieldMapper = docMapper.mappers().smartNameFieldMapper(field); if (fieldMapper == null) { if (docMapper.objectMappers().get(field) != null) { // Only fail if we know it is a object field, missing paths / fields shouldn't fail. throw new IllegalArgumentException("field [" + field + "] isn't a leaf field"); } - } else if (!fieldMapper.fieldType().stored() && !fieldMapper.isGenerated()) { - if (searchLookup == null) { - searchLookup = new SearchLookup(mapperService, null, new String[]{type}); - LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(docIdAndVersion.context); - searchLookup.source().setSource(source); - leafSearchLookup.setDocument(docIdAndVersion.docId); - } - - List values = searchLookup.source().extractRawValues(field); - if (values.isEmpty() == false) { - value = values; - } - } - - if (value != null) { - if (fields == null) { - fields = new HashMap<>(2); - } - if (value instanceof List) { - fields.put(field, new GetField(field, (List) value)); - } else { - fields.put(field, new GetField(field, Collections.singletonList(value))); - } } } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java index f52123784d1..c418dd5e6e2 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -291,8 +291,4 @@ public class AllFieldMapper extends MetadataFieldMapper { super.doMerge(mergeWith, updateAllTypes); } - @Override - public boolean isGenerated() { - return true; - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index ee54a473998..7bce2ae9abf 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -660,14 +660,4 @@ public abstract class FieldMapper extends Mapper implements Cloneable { } } - /** - * Fields might not be available before indexing, for example _all, token_count,... - * When get is called and these fields are requested, this case needs special treatment. - * - * @return If the field is available before indexing or not. - */ - public boolean isGenerated() { - return false; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 1b18652bce5..7343963f099 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -289,8 +289,4 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { return builder; } - @Override - public boolean isGenerated() { - return true; - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java index fd9f6632fe0..2ed1b544a02 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java @@ -187,9 +187,4 @@ public class LegacyTokenCountFieldMapper extends LegacyIntegerFieldMapper { builder.field("analyzer", analyzer()); } - @Override - public boolean isGenerated() { - return true; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index daa36664f9d..9eeaf4012fa 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -182,9 +182,4 @@ public class TokenCountFieldMapper extends FieldMapper { builder.field("analyzer", analyzer()); } - @Override - public boolean isGenerated() { - return true; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 4fe65dcb777..50583a148d7 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -35,6 +35,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetField; import org.elasticsearch.index.get.GetResult; @@ -44,6 +46,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.StringFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.Uid; @@ -55,8 +58,10 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -191,9 +196,11 @@ public class TermVectorsService { } /* generate term vectors from fetched document fields */ + String[] getFields = validFields.toArray(new String[validFields.size() + 1]); + getFields[getFields.length - 1] = SourceFieldMapper.NAME; GetResult getResult = indexShard.getService().get( - get, request.id(), request.type(), validFields.toArray(Strings.EMPTY_ARRAY), null); - Fields generatedTermVectors = generateTermVectors(indexShard, getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields); + get, request.id(), request.type(), getFields, null); + Fields generatedTermVectors = generateTermVectors(indexShard, getResult.sourceAsMap(), getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields); /* merge with existing Fields */ if (termVectorsByField == null) { @@ -227,17 +234,31 @@ public class TermVectorsService { return selectedFields; } - private static Fields generateTermVectors(IndexShard indexShard, Collection getFields, boolean withOffsets, @Nullable Map perFieldAnalyzer, Set fields) throws IOException { - /* store document in memory index */ - MemoryIndex index = new MemoryIndex(withOffsets); + private static Fields generateTermVectors(IndexShard indexShard, Map source, Collection getFields, boolean withOffsets, @Nullable Map perFieldAnalyzer, Set fields) throws IOException { + Map> values = new HashMap<>(); for (GetField getField : getFields) { String field = getField.getName(); - if (fields.contains(field) == false) { - // some fields are returned even when not asked for, eg. _timestamp - continue; + if (fields.contains(field)) { // some fields are returned even when not asked for, eg. _timestamp + values.put(field, getField.getValues()); } + } + if (source != null) { + for (String field : fields) { + if (values.containsKey(field) == false) { + List v = XContentMapValues.extractRawValues(field, source); + if (v.isEmpty() == false) { + values.put(field, v); + } + } + } + } + + /* store document in memory index */ + MemoryIndex index = new MemoryIndex(withOffsets); + for (Map.Entry> entry : values.entrySet()) { + String field = entry.getKey(); Analyzer analyzer = getAnalyzerAtField(indexShard, field, perFieldAnalyzer); - for (Object text : getField.getValues()) { + for (Object text : entry.getValue()) { index.addField(field, text.toString(), analyzer); } } @@ -270,7 +291,7 @@ public class TermVectorsService { String[] values = doc.getValues(field.name()); getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values))); } - return generateTermVectors(indexShard, getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); + return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true).v2(), getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); } private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 16502ff92b1..05e32bbfdde 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -149,21 +149,21 @@ public class BulkWithUpdatesIT extends ESIntegTestCase { assertThat(bulkResponse.getItems()[2].getResponse().getId(), equalTo("3")); assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L)); - GetResponse getResponse = client().prepareGet().setIndex("test").setType("type1").setId("1").setFields("field").execute() + GetResponse getResponse = client().prepareGet().setIndex("test").setType("type1").setId("1").execute() .actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(2L)); - assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(2L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(2L)); - getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").setFields("field").execute().actionGet(); + getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(2L)); - assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(3L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(3L)); - getResponse = client().prepareGet().setIndex("test").setType("type1").setId("3").setFields("field1").execute().actionGet(); + getResponse = client().prepareGet().setIndex("test").setType("type1").setId("3").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(2L)); - assertThat(getResponse.getField("field1").getValue().toString(), equalTo("test")); + assertThat(getResponse.getSource().get("field1").toString(), equalTo("test")); bulkResponse = client() .prepareBulk() @@ -185,18 +185,18 @@ public class BulkWithUpdatesIT extends ESIntegTestCase { assertThat(bulkResponse.getItems()[2].getResponse().getIndex(), equalTo("test")); assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(3L)); - getResponse = client().prepareGet().setIndex("test").setType("type1").setId("6").setFields("field").execute().actionGet(); + getResponse = client().prepareGet().setIndex("test").setType("type1").setId("6").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(1L)); - assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(0L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(0L)); - getResponse = client().prepareGet().setIndex("test").setType("type1").setId("7").setFields("field").execute().actionGet(); + getResponse = client().prepareGet().setIndex("test").setType("type1").setId("7").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(false)); - getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").setFields("field").execute().actionGet(); + getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(3L)); - assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(4L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(4L)); } public void testBulkVersioning() throws Exception { @@ -325,11 +325,11 @@ public class BulkWithUpdatesIT extends ESIntegTestCase { assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(1)); for (int j = 0; j < 5; j++) { - GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).setFields("counter").execute() + GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).execute() .actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getVersion(), equalTo(1L)); - assertThat(((Number) getResponse.getField("counter").getValue()).longValue(), equalTo(1L)); + assertThat(((Number) getResponse.getSource().get("counter")).longValue(), equalTo(1L)); } } diff --git a/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java index 8d7a9bdca02..d9371df09ae 100644 --- a/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java +++ b/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java @@ -37,7 +37,9 @@ public class AliasedIndexDocumentActionsIT extends DocumentActionsIT { // ignore } logger.info("--> creating index test"); - client().admin().indices().create(createIndexRequest("test1").alias(new Alias("test"))).actionGet(); + client().admin().indices().create(createIndexRequest("test1") + .mapping("type1", "name", "type=keyword,store=true") + .alias(new Alias("test"))).actionGet(); } @Override diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java index 065128af918..9196a16060c 100644 --- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.io.IOException; @@ -50,7 +51,7 @@ import static org.hamcrest.Matchers.nullValue; */ public class DocumentActionsIT extends ESIntegTestCase { protected void createIndex() { - createIndex(getConcreteIndexName()); + ElasticsearchAssertions.assertAcked(prepareCreate(getConcreteIndexName()).addMapping("type1", "name", "type=keyword,store=true")); } protected String getConcreteIndexName() { diff --git a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java index 7a64abc52ae..4aacb672999 100644 --- a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java +++ b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java @@ -114,7 +114,9 @@ public class ExplainActionIT extends ESIntegTestCase { } public void testExplainWithFields() throws Exception { - assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); + assertAcked(prepareCreate("test") + .addMapping("test", "obj1.field1", "type=keyword,store=true", "obj1.field2", "type=keyword,store=true") + .addAlias(new Alias("alias"))); ensureGreen("test"); client().prepareIndex("test", "test", "1") diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index 16ccd9d56b8..57de501d93d 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -61,6 +61,7 @@ public class GetActionIT extends ESIntegTestCase { public void testSimpleGet() { assertAcked(prepareCreate("test") + .addMapping("type1", "field1", "type=keyword,store=true", "field2", "type=keyword,store=true") .setSettings(Settings.builder().put("index.refresh_interval", -1)) .addAlias(new Alias("alias"))); ensureGreen(); @@ -107,7 +108,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2")); - logger.info("--> realtime fetch of field (requires fetching parsing source)"); + logger.info("--> realtime fetch of field"); response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").get(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); @@ -115,7 +116,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1")); assertThat(response.getField("field2"), nullValue()); - logger.info("--> realtime fetch of field & source (requires fetching parsing source)"); + logger.info("--> realtime fetch of field & source"); response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").setFetchSource("field1", null).get(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); @@ -189,6 +190,7 @@ public class GetActionIT extends ESIntegTestCase { public void testSimpleMultiGet() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .addMapping("type1", "field", "type=keyword,store=true") .setSettings(Settings.builder().put("index.refresh_interval", -1))); ensureGreen(); @@ -530,7 +532,7 @@ public class GetActionIT extends ESIntegTestCase { public void testGetFieldsMetaData() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") - .addMapping("my-type1", "_parent", "type=parent") + .addMapping("my-type1", "_parent", "type=parent", "field1", "type=keyword,store=true") .addAlias(new Alias("alias")) .setSettings(Settings.builder().put("index.refresh_interval", -1))); diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index 6bcc88975c1..45f8b7490a6 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -227,10 +227,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { // Check that we can get doc 1 and 2, because we are doing realtime // gets and getting from the primary - GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get(); + GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get(); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); flushAndRefresh(IDX); client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get(); @@ -238,10 +238,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { refresh(); // Check that we can get doc 1 and 2 without realtime - gResp1 = client().prepareGet(IDX, "doc", "1").setRealtime(false).setFields("foo").get(); - gResp2 = client().prepareGet(IDX, "doc", "2").setRealtime(false).setFields("foo").get(); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + gResp1 = client().prepareGet(IDX, "doc", "1").setRealtime(false).get(); + gResp2 = client().prepareGet(IDX, "doc", "2").setRealtime(false).get(); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); logger.info("--> restarting all nodes"); if (randomBoolean()) { @@ -283,12 +283,12 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get(); client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get(); - GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); + GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get(); + GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); // Node1 has the primary, now node2 has the replica String node2 = internalCluster().startNode(nodeSettings); @@ -304,21 +304,21 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get(); assertHitCount(resp, 2); - gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); + gResp1 = client().prepareGet(IDX, "doc", "1").get(); + gResp2 = client().prepareGet(IDX, "doc", "2").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.toString(), gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); client().prepareIndex(IDX, "doc", "1").setSource("foo", "foobar").get(); client().prepareIndex(IDX, "doc", "2").setSource("foo", "foobar").get(); - gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); + gResp1 = client().prepareGet(IDX, "doc", "1").get(); + gResp2 = client().prepareGet(IDX, "doc", "2").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.toString(), gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("foobar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("foobar")); + assertThat(gResp1.getSource().get("foo"), equalTo("foobar")); + assertThat(gResp2.getSource().get("foo"), equalTo("foobar")); } public void testPrimaryRelocation() throws Exception { @@ -340,12 +340,12 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get(); client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get(); - GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); + GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get(); + GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); // Node1 has the primary, now node2 has the replica String node2 = internalCluster().startNode(nodeSettings); @@ -363,21 +363,21 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get(); assertHitCount(resp, 2); - gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); + gResp1 = client().prepareGet(IDX, "doc", "1").get(); + gResp2 = client().prepareGet(IDX, "doc", "2").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.toString(), gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get(); client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get(); - gResp1 = client().prepareGet(IDX, "doc", "3").setPreference("_primary").setFields("foo").get(); - gResp2 = client().prepareGet(IDX, "doc", "4").setPreference("_primary").setFields("foo").get(); + gResp1 = client().prepareGet(IDX, "doc", "3").setPreference("_primary").get(); + gResp2 = client().prepareGet(IDX, "doc", "4").setPreference("_primary").get(); assertTrue(gResp1.isExists()); assertTrue(gResp2.isExists()); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); } public void testPrimaryRelocationWithConcurrentIndexing() throws Exception { @@ -573,10 +573,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get(); flushAndRefresh(IDX); - GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").setFields("foo").get(); - GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").setFields("foo").get(); - assertThat(gResp1.getField("foo").getValue().toString(), equalTo("bar")); - assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); + GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get(); + GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get(); + assertThat(gResp1.getSource().get("foo"), equalTo("bar")); + assertThat(gResp2.getSource().get("foo"), equalTo("bar")); logger.info("--> performing query"); SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get(); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index a31692a6bac..418d81fcf1e 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -162,9 +162,4 @@ public class Murmur3FieldMapper extends FieldMapper { } } - @Override - public boolean isGenerated() { - return true; - } - } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml index 15530b8be3d..3de8ab1d91d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/20_fields.yaml @@ -1,6 +1,20 @@ --- "Fields": + - do: + indices.create: + index: test_1 + body: + mappings: + test: + properties: + foo: + type: keyword + store: true + count: + type: integer + store: true + - do: index: index: test_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml index 03572bbbc36..2c85a2d54ab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yaml @@ -1,5 +1,17 @@ --- "Source filtering": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + properties: + count: + type: integer + store: true + - do: index: index: test_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml index f56859ec3cb..2216c6540b8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_fields.yaml @@ -1,6 +1,20 @@ --- "Fields": + - do: + indices.create: + index: test_1 + body: + mappings: + test: + properties: + foo: + type: keyword + store: true + count: + type: integer + store: true + - do: index: index: test_1