From 9c011d5b79848489cda5d3d0e34aa6493a0e88b3 Mon Sep 17 00:00:00 2001
From: kimchy <kimchy@gmail.com>
Date: Tue, 8 Feb 2011 23:40:49 +0200
Subject: [PATCH] improve get perf by using bloom filter to fetch the doc

---
 .../action/get/TransportGetAction.java        | 35 +++++++++++++++----
 .../common/lucene/uid/UidField.java           | 14 ++++----
 .../elasticsearch/index/cache/IndexCache.java |  4 +++
 3 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 2fe4f29a16f..7779022b1e7 100644
--- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -22,22 +22,25 @@ package org.elasticsearch.action.get;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.UnicodeUtil;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.action.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportShardSingleOperationAction;
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.common.Unicode;
+import org.elasticsearch.common.bloom.BloomFilter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.UidField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.cache.bloom.BloomCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.FieldMappers;
-import org.elasticsearch.index.mapper.FieldMappersFieldSelector;
+import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.indices.IndicesService;
@@ -80,6 +83,7 @@ public class TransportGetAction extends TransportShardSingleOperationAction<GetR
 
     @Override protected GetResponse shardOperation(GetRequest request, int shardId) throws ElasticSearchException {
         IndexService indexService = indicesService.indexServiceSafe(request.index());
+        BloomCache bloomCache = indexService.cache().bloomCache();
         IndexShard indexShard = indexService.shardSafe(shardId);
 
         DocumentMapper docMapper = indexService.mapperService().documentMapper(request.type());
@@ -97,15 +101,15 @@ public class TransportGetAction extends TransportShardSingleOperationAction<GetR
         Map<String, GetField> fields = null;
         long version = -1;
         try {
-            UidField.DocIdAndVersion docIdAndVersion = UidField.loadDocIdAndVersion(searcher.reader(), docMapper.uidMapper().term(request.type(), request.id()));
-            if (docIdAndVersion.docId != Lucene.NO_DOC) {
+            UidField.DocIdAndVersion docIdAndVersion = loadCurrentVersionFromIndex(bloomCache, searcher, docMapper.uidMapper().term(request.type(), request.id()));
+            if (docIdAndVersion != null && docIdAndVersion.docId != Lucene.NO_DOC) {
                 if (docIdAndVersion.version > 0) {
                     version = docIdAndVersion.version;
                 }
                 exists = true;
                 FieldSelector fieldSelector = buildFieldSelectors(docMapper, request.fields());
                 if (fieldSelector != null) {
-                    Document doc = searcher.reader().document(docIdAndVersion.docId, fieldSelector);
+                    Document doc = docIdAndVersion.reader.document(docIdAndVersion.docId, fieldSelector);
                     source = extractSource(doc, docMapper);
 
                     for (Object oField : doc.getFields()) {
@@ -180,6 +184,23 @@ public class TransportGetAction extends TransportShardSingleOperationAction<GetR
         return source;
     }
 
+    private UidField.DocIdAndVersion loadCurrentVersionFromIndex(BloomCache bloomCache, Engine.Searcher searcher, Term uid) {
+        UnicodeUtil.UTF8Result utf8 = Unicode.fromStringAsUtf8(uid.text());
+        for (IndexReader reader : searcher.searcher().subReaders()) {
+            BloomFilter filter = bloomCache.filter(reader, UidFieldMapper.NAME, true);
+            // we know that its not there...
+            if (!filter.isPresent(utf8.result, 0, utf8.length)) {
+                continue;
+            }
+            UidField.DocIdAndVersion docIdAndVersion = UidField.loadDocIdAndVersion(reader, uid);
+            // either -2 (its there, but no version associated), or an actual version
+            if (docIdAndVersion.docId != -1) {
+                return docIdAndVersion;
+            }
+        }
+        return null;
+    }
+
     @Override protected GetRequest newRequest() {
         return new GetRequest();
     }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java
index 5ded5ec053c..7d38f59392c 100644
--- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java
+++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java
@@ -43,10 +43,12 @@ public class UidField extends AbstractField {
     public static class DocIdAndVersion {
         public final int docId;
         public final long version;
+        public final IndexReader reader;
 
-        public DocIdAndVersion(int docId, long version) {
+        public DocIdAndVersion(int docId, long version, IndexReader reader) {
             this.docId = docId;
             this.version = version;
+            this.reader = reader;
         }
     }
 
@@ -56,20 +58,20 @@ public class UidField extends AbstractField {
         try {
             uid = reader.termPositions(term);
             if (!uid.next()) {
-                return new DocIdAndVersion(Lucene.NO_DOC, -1);
+                return new DocIdAndVersion(Lucene.NO_DOC, -1, reader);
             }
             docId = uid.doc();
             uid.nextPosition();
             if (!uid.isPayloadAvailable()) {
-                return new DocIdAndVersion(docId, -2);
+                return new DocIdAndVersion(docId, -2, reader);
             }
             if (uid.getPayloadLength() < 8) {
-                return new DocIdAndVersion(docId, -2);
+                return new DocIdAndVersion(docId, -2, reader);
             }
             byte[] payload = uid.getPayload(new byte[8], 0);
-            return new DocIdAndVersion(docId, Numbers.bytesToLong(payload));
+            return new DocIdAndVersion(docId, Numbers.bytesToLong(payload), reader);
         } catch (Exception e) {
-            return new DocIdAndVersion(docId, -2);
+            return new DocIdAndVersion(docId, -2, reader);
         } finally {
             if (uid != null) {
                 try {
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/IndexCache.java
index 12f1e95ea87..e57db9846d8 100644
--- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/IndexCache.java
+++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -84,6 +84,10 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
         return this.idCache;
     }
 
+    public BloomCache bloomCache() {
+        return this.bloomCache;
+    }
+
     public QueryParserCache queryParserCache() {
         return this.queryParserCache;
     }