Mapping: By default, don't index _id, closes #868.

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java

@@ -69,6 +69,11 @@ public final class Uid {
         return type + DELIMITER;
     }
 
+    public static String idFromUid(String uid) {
+        int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can
+        return uid.substring(delimiterIndex + 1);
+    }
+
     public static Uid createUid(String uid) {
         int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can
         return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1));

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IdFieldMapper.java

@@ -39,7 +39,7 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements org.el
     public static class Defaults extends AbstractFieldMapper.Defaults {
         public static final String NAME = "_id";
         public static final String INDEX_NAME = "_id";
-        public static final Field.Index INDEX = Field.Index.NOT_ANALYZED;
+        public static final Field.Index INDEX = Field.Index.NO;
         public static final Field.Store STORE = Field.Store.NO;
         public static final boolean OMIT_NORMS = true;
         public static final boolean OMIT_TERM_FREQ_AND_POSITIONS = true;
@@ -62,11 +62,15 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements org.el
     }
 
     protected IdFieldMapper() {
-        this(Defaults.NAME, Defaults.INDEX_NAME);
+        this(Defaults.NAME, Defaults.INDEX_NAME, Defaults.INDEX);
     }
 
-    protected IdFieldMapper(String name, String indexName) {
+    protected IdFieldMapper(Field.Index index) {
-        this(name, indexName, Defaults.INDEX, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST,
+        this(Defaults.NAME, Defaults.INDEX_NAME, index);
+    }
+
+    protected IdFieldMapper(String name, String indexName, Field.Index index) {
+        this(name, indexName, index, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST,
                 Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS);
     }
 

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java

@@ -21,7 +21,9 @@ package org.elasticsearch.index.mapper.xcontent;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.search.Filter;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Preconditions;
 import org.elasticsearch.common.collect.ImmutableMap;
@@ -30,6 +32,7 @@ import org.elasticsearch.common.compress.lzf.LZF;
 import org.elasticsearch.common.io.stream.BytesStreamInput;
 import org.elasticsearch.common.io.stream.CachedStreamInput;
 import org.elasticsearch.common.io.stream.LZFStreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.thread.ThreadLocals;
 import org.elasticsearch.common.xcontent.*;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -80,9 +83,15 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
 
         private XContentMapper.BuilderContext builderContext = new XContentMapper.BuilderContext(new ContentPath(1));
 
-        public Builder(String index, RootObjectMapper.Builder builder) {
+        public Builder(String index, @Nullable Settings indexSettings, RootObjectMapper.Builder builder) {
             this.index = index;
             this.rootObjectMapper = builder.build(builderContext);
+            if (indexSettings != null) {
+                String idIndexed = indexSettings.get("index.mapping._id.indexed");
+                if (idIndexed != null && Booleans.parseBoolean(idIndexed, false)) {
+                    idFieldMapper = new IdFieldMapper(Field.Index.NOT_ANALYZED);
+                }
+            }
         }
 
         public Builder meta(ImmutableMap<String, Object> meta) {

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java

@@ -132,7 +132,7 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme
 
         XContentMapper.TypeParser.ParserContext parserContext = new XContentMapper.TypeParser.ParserContext(analysisService, typeParsers);
 
-        XContentDocumentMapper.Builder docBuilder = doc(index.name(), (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
+        XContentDocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
 
         for (Map.Entry<String, Object> entry : mapping.entrySet()) {
             String fieldName = Strings.toUnderscoreCase(entry.getKey());

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.mapper.xcontent;
 
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.xcontent.ip.IpFieldMapper;
 
 /**
@@ -31,7 +33,11 @@ public final class XContentMapperBuilders {
     }
 
     public static XContentDocumentMapper.Builder doc(String index, RootObjectMapper.Builder objectBuilder) {
-        return new XContentDocumentMapper.Builder(index, objectBuilder);
+        return new XContentDocumentMapper.Builder(index, null, objectBuilder);
+    }
+
+    public static XContentDocumentMapper.Builder doc(String index, @Nullable Settings settings, RootObjectMapper.Builder objectBuilder) {
+        return new XContentDocumentMapper.Builder(index, settings, objectBuilder);
     }
 
     public static SourceFieldMapper.Builder source() {

modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java

@@ -399,7 +399,11 @@ public class PercolatorExecutor extends AbstractIndexComponent {
         }
 
         @Override public void collect(int doc) throws IOException {
-            String id = fieldData.stringValue(doc);
+            String uid = fieldData.stringValue(doc);
+            if (uid == null) {
+                return;
+            }
+            String id = Uid.idFromUid(uid);
             Query query = queries.get(id);
             if (query == null) {
                 // log???
@@ -417,7 +421,8 @@ public class PercolatorExecutor extends AbstractIndexComponent {
         }
 
         @Override public void setNextReader(IndexReader reader, int docBase) throws IOException {
-            fieldData = percolatorIndex.cache().fieldData().cache(FieldDataType.DefaultTypes.STRING, reader, IdFieldMapper.NAME);
+            // we use the UID because id might not be indexed
+            fieldData = percolatorIndex.cache().fieldData().cache(FieldDataType.DefaultTypes.STRING, reader, UidFieldMapper.NAME);
         }
 
         @Override public boolean acceptsDocsOutOfOrder() {

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleMapperTests.java

@@ -70,7 +70,6 @@ public class SimpleMapperTests {
         assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1")));
         assertThat((double) doc.getBoost(), closeTo(3.7, 0.01));
         assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay"));
-        assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1));
 //        System.out.println("Document: " + doc);
 //        System.out.println("Json: " + docMapper.sourceMapper().value(doc));
     }
@@ -86,7 +85,6 @@ public class SimpleMapperTests {
         assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1")));
         assertThat((double) doc.getBoost(), closeTo(3.7, 0.01));
         assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay"));
-        assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1));
 //        System.out.println("Document: " + doc);
 //        System.out.println("Json: " + docMapper.sourceMapper().value(doc));
     }
@@ -99,7 +97,6 @@ public class SimpleMapperTests {
         assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1")));
         assertThat((double) doc.getBoost(), closeTo(3.7, 0.01));
         assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay"));
-        assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1));
 //        System.out.println("Document: " + doc);
 //        System.out.println("Json: " + docMapper.sourceMapper().value(doc));
     }

modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java

@@ -82,7 +82,7 @@ public class SimpleChildQuerySearchTests extends AbstractNodesTests {
 
         // TEST FETCHING _parent from child
         SearchResponse searchResponse = client.prepareSearch("test")
-                .setQuery(termQuery("child._id", "c1"))
+                .setQuery(idsQuery("child").ids("c1"))
                 .addFields("_parent")
                 .execute().actionGet();
         if (searchResponse.failedShards() > 0) {

modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/compress/SearchSourceCompressTests.java

@@ -99,7 +99,7 @@ public class SearchSourceCompressTests extends AbstractNodesTests {
         assertThat(getResponse.source(), equalTo(buildSource(10000).copiedBytes()));
 
         for (int i = 1; i < 100; i++) {
-            SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.termQuery("_id", Integer.toString(i))).execute().actionGet();
+            SearchResponse searchResponse = client.prepareSearch().setQuery(QueryBuilders.idsQuery("type1").ids(Integer.toString(i))).execute().actionGet();
             assertThat(searchResponse.hits().getTotalHits(), equalTo(1l));
             assertThat(searchResponse.hits().getAt(0).source(), equalTo(buildSource(i).copiedBytes()));
         }