From 3af95913a90a3a8366ef04694a87b196e1f922c9 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Wed, 6 Jun 2018 18:03:34 -0700
Subject: [PATCH] Lazy-ify IncrementalIndex filtering too. (#5852)

* Lazy-ify IncrementalIndex filtering too.

Follow-up to #5403, which only lazy-ified cursor-based filtering
on QueryableIndex.

* Fix logic error.
---
 .../druid/segment/DimensionSelectorUtils.java |  2 +-
 .../druid/segment/StringDimensionIndexer.java | 22 ++++++++++++++-----
 .../column/SimpleDictionaryEncodedColumn.java |  2 +-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java
index 092675c8551..b1df53cd904 100644
--- a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java
+++ b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java
@@ -52,7 +52,7 @@ public final class DimensionSelectorUtils
     if (idLookup != null) {
       return makeDictionaryEncodedValueMatcherGeneric(selector, idLookup.lookupId(value), value == null);
     } else if (selector.getValueCardinality() >= 0 && selector.nameLookupPossibleInAdvance()) {
-      // Employ precomputed BitSet optimization
+      // Employ caching BitSet optimization
       return makeDictionaryEncodedValueMatcherGeneric(selector, Predicates.equalTo(value));
     } else {
       return makeNonDictionaryEncodedValueMatcherGeneric(selector, value);
diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java
index df28aa43f2e..e36b223731f 100644
--- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java
+++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java
@@ -177,7 +177,7 @@ public class StringDimensionIndexer implements DimensionIndexer<Integer, int[],
       this.idToIndex = new int[length];
       this.indexToId = new int[length];
       int index = 0;
-      for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext();) {
+      for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext(); ) {
         int id = iterator.nextInt();
         idToIndex[id] = index;
         indexToId[index] = id;
@@ -496,7 +496,7 @@ public class StringDimensionIndexer implements DimensionIndexer<Integer, int[],
             return BooleanValueMatcher.of(false);
           }
         } else {
-          // Employ precomputed BitSet optimization
+          // Employ caching BitSet optimization
           return makeValueMatcher(Predicates.equalTo(value));
         }
       }
@@ -504,8 +504,11 @@ public class StringDimensionIndexer implements DimensionIndexer<Integer, int[],
       @Override
       public ValueMatcher makeValueMatcher(final Predicate<String> predicate)
       {
-        final BitSet predicateMatchingValueIds = DimensionSelectorUtils.makePredicateMatchingSet(this, predicate);
+        final BitSet checkedIds = new BitSet(maxId);
+        final BitSet matchingIds = new BitSet(maxId);
         final boolean matchNull = predicate.apply(null);
+
+        // Lazy matcher; only check an id if matches() is called.
         return new ValueMatcher()
         {
           @Override
@@ -522,8 +525,17 @@ public class StringDimensionIndexer implements DimensionIndexer<Integer, int[],
             }
 
             for (int id : dimsInt) {
-              if (predicateMatchingValueIds.get(id)) {
-                return true;
+              if (checkedIds.get(id)) {
+                if (matchingIds.get(id)) {
+                  return true;
+                }
+              } else {
+                final boolean matches = predicate.apply(lookupName(id));
+                checkedIds.set(id);
+                if (matches) {
+                  matchingIds.set(id);
+                  return true;
+                }
               }
             }
             return false;
diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java
index ab12164720b..3b3d8fe4f71 100644
--- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java
+++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java
@@ -259,7 +259,7 @@ public class SimpleDictionaryEncodedColumn implements DictionaryEncodedColumn<St
               return BooleanValueMatcher.of(false);
             }
           } else {
-            // Employ precomputed BitSet optimization
+            // Employ caching BitSet optimization
             return makeValueMatcher(Predicates.equalTo(value));
           }
         }