diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/StringDimensionIndexerBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/StringDimensionIndexerBenchmark.java index 487c7ac0b90..e9df76b39c8 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/StringDimensionIndexerBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/StringDimensionIndexerBenchmark.java @@ -32,6 +32,7 @@ import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; @@ -53,7 +54,7 @@ public class StringDimensionIndexerBenchmark @Param({"10000"}) public int cardinality; - @Param({"8"}) + @Param({"8", "40"}) public int rowSize; @Setup @@ -75,9 +76,20 @@ public class StringDimensionIndexerBenchmark @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) + @Threads(1) public void estimateEncodedKeyComponentSize(Blackhole blackhole) { long sz = indexer.estimateEncodedKeyComponentSize(exampleArray); blackhole.consume(sz); } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @Threads(2) + public void estimateEncodedKeyComponentSizeTwoThreads(Blackhole blackhole) + { + long sz = indexer.estimateEncodedKeyComponentSize(exampleArray); + blackhole.consume(sz); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnIndexer.java b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnIndexer.java index a2cbfb76e81..35a08969a23 100644 --- a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnIndexer.java +++ b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnIndexer.java @@ -45,17 +45,6 @@ public abstract class DictionaryEncodedColumnIndexer sortedLookup; - /** - * Creates a new DictionaryEncodedColumnIndexer with the default implementation - * of {@link DimensionDictionary}. - *

- * Using this constructor disables memory estimations of the dictionary size. - */ - public DictionaryEncodedColumnIndexer() - { - this(new DimensionDictionary<>()); - } - /** * Creates a new DictionaryEncodedColumnIndexer. * diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionDictionary.java b/processing/src/main/java/org/apache/druid/segment/DimensionDictionary.java index 17c2cc4ce1d..3c4d0a64f93 100644 --- a/processing/src/main/java/org/apache/druid/segment/DimensionDictionary.java +++ b/processing/src/main/java/org/apache/druid/segment/DimensionDictionary.java @@ -23,6 +23,8 @@ import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import javax.annotation.Nullable; + +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -38,6 +40,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; public class DimensionDictionary> { public static final int ABSENT_VALUE_ID = -1; + private final Class cls; @Nullable private T minValue = null; @@ -51,8 +54,9 @@ public class DimensionDictionary> private final List idToValue = new ArrayList<>(); private final ReentrantReadWriteLock lock; - public DimensionDictionary() + public DimensionDictionary(Class cls) { + this.cls = cls; this.lock = new ReentrantReadWriteLock(); valueToId.defaultReturnValue(ABSENT_VALUE_ID); } @@ -86,6 +90,22 @@ public class DimensionDictionary> } } + public T[] getValues(int[] ids) + { + T[] values = (T[]) Array.newInstance(cls, ids.length); + + lock.readLock().lock(); + try { + for (int i = 0; i < ids.length; i++) { + values[i] = (ids[i] == idForNull) ? null : idToValue.get(ids[i]); + } + return values; + } + finally { + lock.readLock().unlock(); + } + } + public int size() { lock.readLock().lock(); diff --git a/processing/src/main/java/org/apache/druid/segment/StringDimensionDictionary.java b/processing/src/main/java/org/apache/druid/segment/StringDimensionDictionary.java index 282924019d1..ccff9a2589b 100644 --- a/processing/src/main/java/org/apache/druid/segment/StringDimensionDictionary.java +++ b/processing/src/main/java/org/apache/druid/segment/StringDimensionDictionary.java @@ -34,6 +34,7 @@ public class StringDimensionDictionary extends DimensionDictionary */ public StringDimensionDictionary(boolean computeOnHeapSize) { + super(String.class); this.computeOnHeapSize = computeOnHeapSize; } diff --git a/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java index c88f22f477b..589ad84fee3 100644 --- a/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java @@ -148,14 +148,15 @@ public class StringDimensionIndexer extends DictionaryEncodedColumnIndexer