From af39f07213ccce2419688191b47fbac5fbd4de40 Mon Sep 17 00:00:00 2001 From: kimchy Date: Wed, 20 Apr 2011 02:49:14 +0300 Subject: [PATCH] Mapping: By default, don't index _id, closes #868. --- .../java/org/elasticsearch/index/mapper/Uid.java | 5 +++++ .../index/mapper/xcontent/IdFieldMapper.java | 12 ++++++++---- .../mapper/xcontent/XContentDocumentMapper.java | 11 ++++++++++- .../xcontent/XContentDocumentMapperParser.java | 2 +- .../mapper/xcontent/XContentMapperBuilders.java | 8 +++++++- .../index/percolator/PercolatorExecutor.java | 9 +++++++-- .../mapper/xcontent/simple/SimpleMapperTests.java | 3 --- .../search/child/SimpleChildQuerySearchTests.java | 2 +- .../search/compress/SearchSourceCompressTests.java | 2 +- 9 files changed, 40 insertions(+), 14 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java index 313e21d9058..922f2abcfbb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java @@ -69,6 +69,11 @@ public final class Uid { return type + DELIMITER; } + public static String idFromUid(String uid) { + int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can + return uid.substring(delimiterIndex + 1); + } + public static Uid createUid(String uid) { int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1)); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IdFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IdFieldMapper.java index 002d9ea8e01..fe8a66adc04 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IdFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IdFieldMapper.java @@ -39,7 +39,7 @@ public class IdFieldMapper extends AbstractFieldMapper implements org.el public static class Defaults extends AbstractFieldMapper.Defaults { public static final String NAME = "_id"; public static final String INDEX_NAME = "_id"; - public static final Field.Index INDEX = Field.Index.NOT_ANALYZED; + public static final Field.Index INDEX = Field.Index.NO; public static final Field.Store STORE = Field.Store.NO; public static final boolean OMIT_NORMS = true; public static final boolean OMIT_TERM_FREQ_AND_POSITIONS = true; @@ -62,11 +62,15 @@ public class IdFieldMapper extends AbstractFieldMapper implements org.el } protected IdFieldMapper() { - this(Defaults.NAME, Defaults.INDEX_NAME); + this(Defaults.NAME, Defaults.INDEX_NAME, Defaults.INDEX); } - protected IdFieldMapper(String name, String indexName) { - this(name, indexName, Defaults.INDEX, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST, + protected IdFieldMapper(Field.Index index) { + this(Defaults.NAME, Defaults.INDEX_NAME, index); + } + + protected IdFieldMapper(String name, String indexName, Field.Index index) { + this(name, indexName, index, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java index e5c1676df26..41b762a5241 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java @@ -21,7 +21,9 @@ package org.elasticsearch.index.mapper.xcontent; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.search.Filter; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.collect.ImmutableMap; @@ -30,6 +32,7 @@ import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.CachedStreamInput; import org.elasticsearch.common.io.stream.LZFStreamInput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.thread.ThreadLocals; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -80,9 +83,15 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent { private XContentMapper.BuilderContext builderContext = new XContentMapper.BuilderContext(new ContentPath(1)); - public Builder(String index, RootObjectMapper.Builder builder) { + public Builder(String index, @Nullable Settings indexSettings, RootObjectMapper.Builder builder) { this.index = index; this.rootObjectMapper = builder.build(builderContext); + if (indexSettings != null) { + String idIndexed = indexSettings.get("index.mapping._id.indexed"); + if (idIndexed != null && Booleans.parseBoolean(idIndexed, false)) { + idFieldMapper = new IdFieldMapper(Field.Index.NOT_ANALYZED); + } + } } public Builder meta(ImmutableMap meta) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java index e69af849e0d..f3b9aedb2ea 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java @@ -132,7 +132,7 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme XContentMapper.TypeParser.ParserContext parserContext = new XContentMapper.TypeParser.ParserContext(analysisService, typeParsers); - XContentDocumentMapper.Builder docBuilder = doc(index.name(), (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext)); + XContentDocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext)); for (Map.Entry entry : mapping.entrySet()) { String fieldName = Strings.toUnderscoreCase(entry.getKey()); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java index 75447a2f3c0..ddf96fd7a5e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper.xcontent; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.xcontent.ip.IpFieldMapper; /** @@ -31,7 +33,11 @@ public final class XContentMapperBuilders { } public static XContentDocumentMapper.Builder doc(String index, RootObjectMapper.Builder objectBuilder) { - return new XContentDocumentMapper.Builder(index, objectBuilder); + return new XContentDocumentMapper.Builder(index, null, objectBuilder); + } + + public static XContentDocumentMapper.Builder doc(String index, @Nullable Settings settings, RootObjectMapper.Builder objectBuilder) { + return new XContentDocumentMapper.Builder(index, settings, objectBuilder); } public static SourceFieldMapper.Builder source() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java index 249d39353e4..31461eb6367 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java @@ -399,7 +399,11 @@ public class PercolatorExecutor extends AbstractIndexComponent { } @Override public void collect(int doc) throws IOException { - String id = fieldData.stringValue(doc); + String uid = fieldData.stringValue(doc); + if (uid == null) { + return; + } + String id = Uid.idFromUid(uid); Query query = queries.get(id); if (query == null) { // log??? @@ -417,7 +421,8 @@ public class PercolatorExecutor extends AbstractIndexComponent { } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { - fieldData = percolatorIndex.cache().fieldData().cache(FieldDataType.DefaultTypes.STRING, reader, IdFieldMapper.NAME); + // we use the UID because id might not be indexed + fieldData = percolatorIndex.cache().fieldData().cache(FieldDataType.DefaultTypes.STRING, reader, UidFieldMapper.NAME); } @Override public boolean acceptsDocsOutOfOrder() { diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleMapperTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleMapperTests.java index 6350c8b8f3f..43a356dfac5 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleMapperTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleMapperTests.java @@ -70,7 +70,6 @@ public class SimpleMapperTests { assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat((double) doc.getBoost(), closeTo(3.7, 0.01)); assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay")); - assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1)); // System.out.println("Document: " + doc); // System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } @@ -86,7 +85,6 @@ public class SimpleMapperTests { assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat((double) doc.getBoost(), closeTo(3.7, 0.01)); assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay")); - assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1)); // System.out.println("Document: " + doc); // System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } @@ -99,7 +97,6 @@ public class SimpleMapperTests { assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat((double) doc.getBoost(), closeTo(3.7, 0.01)); assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay")); - assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1)); // System.out.println("Document: " + doc); // System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java index 3453eda93da..edc48b69ec2 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java @@ -82,7 +82,7 @@ public class SimpleChildQuerySearchTests extends AbstractNodesTests { // TEST FETCHING _parent from child SearchResponse searchResponse = client.prepareSearch("test") - .setQuery(termQuery("child._id", "c1")) + .setQuery(idsQuery("child").ids("c1")) .addFields("_parent") .execute().actionGet(); if (searchResponse.failedShards() > 0) { diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/compress/SearchSourceCompressTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/compress/SearchSourceCompressTests.java index dbbbca32884..780cdaa7958 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/compress/SearchSourceCompressTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/compress/SearchSourceCompressTests.java @@ -99,7 +99,7 @@ public class SearchSourceCompressTests extends AbstractNodesTests { assertThat(getResponse.source(), equalTo(buildSource(10000).copiedBytes())); for (int i = 1; i < 100; i++) { - SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.termQuery("_id", Integer.toString(i))).execute().actionGet(); + SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.idsQuery("type1").ids(Integer.toString(i))).execute().actionGet(); assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().getAt(0).source(), equalTo(buildSource(i).copiedBytes())); }