diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index d41589142d7..d719da7dba3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -165,7 +165,13 @@ public class SqlNestedDataBenchmark "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46)", // 24, 25 "SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1", - "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1" + "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1", + // 26, 27 + "SELECT SUM(long1) FROM foo WHERE double3 < 1005.0 AND double3 > 1000.0", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 1005.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0", + // 28, 29 + "SELECT SUM(long1) FROM foo WHERE double3 < 2000.0 AND double3 > 1000.0", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0" ); @Param({"5000000"}) @@ -203,7 +209,11 @@ public class SqlNestedDataBenchmark "22", "23", "24", - "25" + "25", + "26", + "27", + "28", + "29" }) private String query; diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index 74270b65c5c..60e07ee09b2 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -523,7 +523,7 @@ public class InDimFilter extends AbstractOptimizableDimFilter implements Filter private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; - InFilterDruidPredicateFactory( + public InFilterDruidPredicateFactory( final ExtractionFn extractionFn, final ValuesSet values ) diff --git a/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java new file mode 100644 index 00000000000..bb9bf5c6254 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import javax.annotation.Nullable; + +/** + * An optimized column value {@link BitmapColumnIndex} provider for specialized processing of numeric value ranges. + * This index does not match null values, union the results of this index with {@link NullValueIndex} if null values + * should be considered part of the value range. + */ +public interface NumericRangeIndex +{ + /** + * Get a {@link BitmapColumnIndex} corresponding to the values supplied in the specified range. If supplied starting + * value is null, the range will begin at the first non-null value in the underlying value dictionary. If the end + * value is null, the range will extend to the last value in the underlying value dictionary. + */ + BitmapColumnIndex forRange( + @Nullable Number startValue, + boolean startStrict, + @Nullable Number endValue, + boolean endStrict + ); +} diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 0c00d310864..b4c1463b078 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -48,6 +48,7 @@ import org.apache.druid.segment.column.ColumnIndexCapabilities; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.LexicographicalRangeIndex; import org.apache.druid.segment.column.NullValueIndex; +import org.apache.druid.segment.column.NumericRangeIndex; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -75,71 +76,107 @@ public class BoundFilter implements Filter if (!Filters.checkFilterTuningUseIndex(boundDimFilter.getDimension(), selector, filterTuning)) { return null; } - if (supportShortCircuit()) { + if (supportStringShortCircuit()) { final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(boundDimFilter.getDimension()); if (indexSupplier == null) { return Filters.makeNullIndex(doesMatchNull(), selector); } final LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class); - if (rangeIndex == null) { - // column - return null; - } - final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange( - boundDimFilter.getLower(), - boundDimFilter.isLowerStrict(), - boundDimFilter.getUpper(), - boundDimFilter.isUpperStrict() - ); - // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set - if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { - return rangeBitmaps; - } else { - final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class); - if (nulls == null) { - return null; + if (rangeIndex != null) { + final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange( + boundDimFilter.getLower(), + boundDimFilter.isLowerStrict(), + boundDimFilter.getUpper(), + boundDimFilter.isUpperStrict() + ); + // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set + if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { + return rangeBitmaps; + } else { + return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); } - final BitmapColumnIndex nullBitmap = nulls.forNull(); - return new BitmapColumnIndex() - { - @Override - public ColumnIndexCapabilities getIndexCapabilities() - { - return rangeBitmaps.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities()); - } - - @Override - public double estimateSelectivity(int totalRows) - { - return Math.min( - 1.0, - rangeBitmaps.estimateSelectivity(totalRows) + nullBitmap.estimateSelectivity(totalRows) - ); - } - - @Override - public T computeBitmapResult(BitmapResultFactory bitmapResultFactory) - { - return bitmapResultFactory.union( - ImmutableList.of( - rangeBitmaps.computeBitmapResult(bitmapResultFactory), - nullBitmap.computeBitmapResult(bitmapResultFactory) - ) - ); - } - }; } - } else { - return Filters.makePredicateIndex(boundDimFilter.getDimension(), selector, getPredicateFactory()); } + if (supportNumericShortCircuit()) { + final ColumnIndexSupplier indexSupplier = selector.getIndexSupplier(boundDimFilter.getDimension()); + if (indexSupplier == null) { + return Filters.makeNullIndex(doesMatchNull(), selector); + } + final NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class); + if (rangeIndex != null) { + final Number lower = boundDimFilter.hasLowerBound() ? Double.parseDouble(boundDimFilter.getLower()) : null; + final Number upper = boundDimFilter.hasUpperBound() ? Double.parseDouble(boundDimFilter.getUpper()) : null; + final BitmapColumnIndex rangeBitmaps = rangeIndex.forRange( + lower, + boundDimFilter.isLowerStrict(), + upper, + boundDimFilter.isUpperStrict() + ); + // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set + if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { + return rangeBitmaps; + } else { + return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); + } + } + } + // fall back to predicate based index if it is available + return Filters.makePredicateIndex(boundDimFilter.getDimension(), selector, getPredicateFactory()); } - private boolean supportShortCircuit() + @Nullable + private BitmapColumnIndex wrapRangeIndexWithNullValueIndex( + ColumnIndexSupplier indexSupplier, + BitmapColumnIndex rangeIndex + ) + { + final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class); + if (nulls == null) { + return null; + } + final BitmapColumnIndex nullBitmap = nulls.forNull(); + return new BitmapColumnIndex() + { + @Override + public ColumnIndexCapabilities getIndexCapabilities() + { + return rangeIndex.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities()); + } + + @Override + public double estimateSelectivity(int totalRows) + { + return Math.min( + 1.0, + rangeIndex.estimateSelectivity(totalRows) + nullBitmap.estimateSelectivity(totalRows) + ); + } + + @Override + public T computeBitmapResult(BitmapResultFactory bitmapResultFactory) + { + return bitmapResultFactory.union( + ImmutableList.of( + rangeIndex.computeBitmapResult(bitmapResultFactory), + nullBitmap.computeBitmapResult(bitmapResultFactory) + ) + ); + } + }; + } + + private boolean supportStringShortCircuit() { // Optimization for lexicographic bounds with no extractionFn => binary search through the index return boundDimFilter.getOrdering().equals(StringComparators.LEXICOGRAPHIC) && extractionFn == null; } + private boolean supportNumericShortCircuit() + { + // Optimization for numeric bounds with no extractionFn => binary search through the index + return boundDimFilter.getOrdering().equals(StringComparators.NUMERIC) && extractionFn == null; + } + @Override public ValueMatcher makeMatcher(ColumnSelectorFactory factory) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java index 99df4d99d5e..e67ac8ac528 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplier.java @@ -35,23 +35,27 @@ import it.unimi.dsi.fastutil.longs.LongIterator; import it.unimi.dsi.fastutil.longs.LongSet; import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.guava.GuavaUtils; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.DruidDoublePredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.segment.IntListUtils; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DruidPredicateIndex; import org.apache.druid.segment.column.LexicographicalRangeIndex; import org.apache.druid.segment.column.NullValueIndex; +import org.apache.druid.segment.column.NumericRangeIndex; import org.apache.druid.segment.column.SimpleBitmapColumnIndex; import org.apache.druid.segment.column.SimpleImmutableBitmapIndex; import org.apache.druid.segment.column.SimpleImmutableBitmapIterableIndex; import org.apache.druid.segment.column.StringValueSetIndex; import org.apache.druid.segment.data.FixedIndexed; import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.Indexed; import javax.annotation.Nullable; import java.util.Iterator; @@ -120,6 +124,8 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie case LONG: if (clazz.equals(StringValueSetIndex.class)) { return (T) new NestedLongLiteralValueSetIndex(); + } else if (clazz.equals(NumericRangeIndex.class)) { + return (T) new NestedLongLiteralNumericRangeIndex(); } else if (clazz.equals(DruidPredicateIndex.class)) { return (T) new NestedLongLiteralPredicateIndex(); } @@ -127,6 +133,8 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie case DOUBLE: if (clazz.equals(StringValueSetIndex.class)) { return (T) new NestedDoubleLiteralValueSetIndex(); + } else if (clazz.equals(NumericRangeIndex.class)) { + return (T) new NestedDoubleLiteralNumericRangeIndex(); } else if (clazz.equals(DruidPredicateIndex.class)) { return (T) new NestedDoubleLiteralPredicateIndex(); } @@ -153,47 +161,106 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie return bitmap == null ? bitmapFactory.makeEmptyImmutableBitmap() : bitmap; } - private IntIntPair getGlobalRange( - @Nullable String startValue, + /** + * Gets a value range from a global dictionary and maps it to a range on the local {@link #dictionary}. + * The starting index of the resulting range is inclusive, while the endpoint is exclusive [start, end) + */ + private IntIntPair getLocalRangeFromDictionary( + @Nullable T startValue, boolean startStrict, - @Nullable String endValue, + @Nullable T endValue, boolean endStrict, - int rangeStart, - int rangeEnd, - GlobalIndexGetFunction getFn + Indexed globalDictionary, + int adjust ) { - int startIndex, endIndex; + int globalStartIndex, globalEndIndex; + int localStartIndex, localEndIndex; if (startValue == null) { - startIndex = rangeStart; + globalStartIndex = adjust == 0 ? 1 : adjust; // global index 0 is always the null value } else { - final int found = getFn.indexOf(startValue); + final int found = globalDictionary.indexOf(startValue); if (found >= 0) { - startIndex = startStrict ? found + 1 : found; + globalStartIndex = adjust + (startStrict ? found + 1 : found); } else { - startIndex = -(found + 1); + globalStartIndex = adjust + (-(found + 1)); } } + // with starting global index settled, now lets find starting local index + int localFound = dictionary.indexOf(globalStartIndex); + if (localFound < 0) { + // the first valid global index is not within the local dictionary, so the insertion point is where we begin + localStartIndex = -(localFound + 1); + } else { + // valid global index in local dictionary, start here + localStartIndex = localFound; + } if (endValue == null) { - endIndex = rangeEnd; + globalEndIndex = globalDictionary.size() + adjust; } else { - final int found = getFn.indexOf(endValue); + final int found = globalDictionary.indexOf(endValue); if (found >= 0) { - endIndex = endStrict ? found : found + 1; + globalEndIndex = adjust + (endStrict ? found : found + 1); } else { - endIndex = -(found + 1); + globalEndIndex = adjust + (-(found + 1)); } } + globalEndIndex = Math.max(globalStartIndex, globalEndIndex); + // end index is not inclusive, so we find the last value in the local dictionary that falls within the range + int localEndFound = dictionary.indexOf(globalEndIndex - 1); + if (localEndFound < 0) { + localEndIndex = -localEndFound; + } else { + // add 1 because the last valid global end value is in the local dictionary, and end index is exclusive + localEndIndex = localEndFound + 1; + } - endIndex = Math.max(startIndex, endIndex); - return new IntIntImmutablePair(startIndex, endIndex); + return new IntIntImmutablePair(localStartIndex, Math.min(dictionary.size(), localEndIndex)); } - @FunctionalInterface - interface GlobalIndexGetFunction + private BitmapColumnIndex makeRangeIndex( + @Nullable T startValue, + boolean startStrict, + @Nullable T endValue, + boolean endStrict, + Indexed globalDictionary, + int adjust + ) { - int indexOf(String value); + final IntIntPair localRange = getLocalRangeFromDictionary( + startValue, + startStrict, + endValue, + endStrict, + globalDictionary, + adjust + ); + final int startIndex = localRange.leftInt(); + final int endIndex = localRange.rightInt(); + return new SimpleImmutableBitmapIterableIndex() + { + @Override + public Iterable getBitmapIterable() + { + return () -> new Iterator() + { + final IntIterator rangeIterator = IntListUtils.fromTo(startIndex, endIndex).iterator(); + + @Override + public boolean hasNext() + { + return rangeIterator.hasNext(); + } + + @Override + public ImmutableBitmap next() + { + return getBitmap(rangeIterator.nextInt()); + } + }; + } + }; } private class NestedStringLiteralValueSetIndex implements StringValueSetIndex @@ -274,7 +341,6 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie private class NestedStringLiteralLexicographicalRangeIndex implements LexicographicalRangeIndex { - @Override public BitmapColumnIndex forRange( @Nullable String startValue, @@ -283,70 +349,14 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie boolean endStrict ) { - return new SimpleImmutableBitmapIterableIndex() - { - @Override - public Iterable getBitmapIterable() - { - final IntIntPair range = getGlobalRange( - startValue, - startStrict, - endValue, - endStrict, - 0, - globalDictionary.size(), - globalDictionary::indexOf - ); - final int start = range.leftInt(), end = range.rightInt(); - // iterates over the range of values in the global dictionary, mapping to relevant range in the local - // dictionary, skipping duplicates - return () -> new Iterator() - { - int currentGlobalIndex = start; - // initialize to -1 because findNext uses this field to check for duplicates, and could legitimately find - // 0 for the first candidate - @SuppressWarnings("UnusedAssignment") - int currentLocalIndex = -1; - { - currentLocalIndex = findNext(); - } - - private int findNext() - { - int candidateLocalIndex = Math.abs(dictionary.indexOf(currentGlobalIndex)); - while (currentGlobalIndex < end && candidateLocalIndex == currentLocalIndex) { - currentGlobalIndex++; - candidateLocalIndex = Math.abs(dictionary.indexOf(currentGlobalIndex)); - } - if (currentGlobalIndex < end) { - currentGlobalIndex++; - return candidateLocalIndex; - } else { - return -1; - } - } - - @Override - public boolean hasNext() - { - return currentLocalIndex != -1; - } - - @Override - public ImmutableBitmap next() - { - int cur = currentLocalIndex; - - if (cur == -1) { - throw new NoSuchElementException(); - } - - currentLocalIndex = findNext(); - return getBitmap(cur); - } - }; - } - }; + return makeRangeIndex( + NullHandling.emptyToNullIfNeeded(startValue), + startStrict, + NullHandling.emptyToNullIfNeeded(endValue), + endStrict, + globalDictionary, + 0 + ); } @Override @@ -363,67 +373,53 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie @Override public Iterable getBitmapIterable() { - final IntIntPair stringsRange = getGlobalRange( + final IntIntPair range = getLocalRangeFromDictionary( startValue, startStrict, endValue, endStrict, - 0, - globalDictionary.size(), - globalDictionary::indexOf + globalDictionary, + 0 ); - // iterates over the range of values in the global dictionary, mapping to relevant range in the local - // dictionary, skipping duplicates + final int start = range.leftInt(), end = range.rightInt(); return () -> new Iterator() { - int currentGlobalIndex = stringsRange.leftInt(); - final int end = stringsRange.rightInt(); - // initialize to -1 because findNext uses this field to check for duplicates, and could legitimately find - // 0 for the first candidate - @SuppressWarnings("UnusedAssignment") - int currentLocalIndex = -1; + int currIndex = start; + int found; + { - currentLocalIndex = findNext(); + found = findNext(); } private int findNext() { - int candidateLocalIndex = Math.abs(dictionary.indexOf(currentGlobalIndex)); - while (currentGlobalIndex < end && shouldSkipGlobal(candidateLocalIndex)) { - currentGlobalIndex++; - candidateLocalIndex = Math.abs(dictionary.indexOf(currentGlobalIndex)); + while (currIndex < end && !matcher.apply(globalDictionary.get(dictionary.get(currIndex)))) { + currIndex++; } - if (currentGlobalIndex < end) { - currentGlobalIndex++; - return candidateLocalIndex; + if (currIndex < end) { + return currIndex++; } else { return -1; } } - private boolean shouldSkipGlobal(int candidate) - { - return currentLocalIndex == candidate || !matcher.apply(globalDictionary.get(currentGlobalIndex)); - } - - @Override public boolean hasNext() { - return currentLocalIndex != -1; + return found != -1; } @Override public ImmutableBitmap next() { - int cur = currentLocalIndex; + int cur = found; if (cur == -1) { throw new NoSuchElementException(); } - currentLocalIndex = findNext(); + found = findNext(); return getBitmap(cur); } }; @@ -492,10 +488,8 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie } } - private class NestedLongLiteralValueSetIndex implements StringValueSetIndex { - @Override public BitmapColumnIndex forValue(@Nullable String value) { @@ -592,6 +586,27 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie } } + private class NestedLongLiteralNumericRangeIndex implements NumericRangeIndex + { + @Override + public BitmapColumnIndex forRange( + @Nullable Number startValue, + boolean startStrict, + @Nullable Number endValue, + boolean endStrict + ) + { + return makeRangeIndex( + startValue != null ? startValue.longValue() : null, + startStrict, + endValue != null ? endValue.longValue() : null, + endStrict, + globalLongDictionary, + adjustLongId + ); + } + } + private class NestedLongLiteralPredicateIndex implements DruidPredicateIndex { @Override @@ -658,7 +673,6 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie private class NestedDoubleLiteralValueSetIndex implements StringValueSetIndex { - @Override public BitmapColumnIndex forValue(@Nullable String value) { @@ -755,6 +769,27 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie } } + private class NestedDoubleLiteralNumericRangeIndex implements NumericRangeIndex + { + @Override + public BitmapColumnIndex forRange( + @Nullable Number startValue, + boolean startStrict, + @Nullable Number endValue, + boolean endStrict + ) + { + return makeRangeIndex( + startValue != null ? startValue.doubleValue() : null, + startStrict, + endValue != null ? endValue.doubleValue() : null, + endStrict, + globalDoubleDictionary, + adjustDoubleId + ); + } + } + private class NestedDoubleLiteralPredicateIndex implements DruidPredicateIndex { @Override @@ -818,7 +853,7 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie } } - private abstract class NestedAnyLiteralIndex + private abstract class NestedVariantLiteralIndex { IntList getIndexes(@Nullable String value) { @@ -858,7 +893,7 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie /** * {@link StringValueSetIndex} but for variant typed nested literal columns */ - private class NestedVariantLiteralValueSetIndex extends NestedAnyLiteralIndex implements StringValueSetIndex + private class NestedVariantLiteralValueSetIndex extends NestedVariantLiteralIndex implements StringValueSetIndex { @Override public BitmapColumnIndex forValue(@Nullable String value) @@ -937,9 +972,8 @@ public class NestedFieldLiteralColumnIndexSupplier implements ColumnIndexSupplie /** * {@link DruidPredicateIndex} but for variant typed nested literal columns */ - private class NestedVariantLiteralPredicateIndex extends NestedAnyLiteralIndex implements DruidPredicateIndex + private class NestedVariantLiteralPredicateIndex extends NestedVariantLiteralIndex implements DruidPredicateIndex { - @Override public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java index c72afd9009d..13a451428da 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldLiteralColumnIndexSupplierTest.java @@ -19,80 +19,1483 @@ package org.apache.druid.segment.nested; -import junit.framework.TestCase; +import com.google.common.collect.ImmutableSet; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.MutableBitmap; -import org.apache.druid.collections.bitmap.RoaringBitmapFactory; +import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.DefaultBitmapResultFactory; +import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.query.filter.InDimFilter; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.DruidPredicateIndex; import org.apache.druid.segment.column.LexicographicalRangeIndex; +import org.apache.druid.segment.column.NullValueIndex; +import org.apache.druid.segment.column.NumericRangeIndex; +import org.apache.druid.segment.column.StringValueSetIndex; +import org.apache.druid.segment.column.TypeStrategies; +import org.apache.druid.segment.data.BitmapSerdeFactory; import org.apache.druid.segment.data.FixedIndexed; +import org.apache.druid.segment.data.FixedIndexedWriter; import org.apache.druid.segment.data.GenericIndexed; -import org.easymock.EasyMock; +import org.apache.druid.segment.data.GenericIndexedWriter; +import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; +import org.apache.druid.segment.serde.Serializer; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.roaringbitmap.IntIterator; -public class NestedFieldLiteralColumnIndexSupplierTest extends TestCase +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; +import java.util.TreeSet; + +public class NestedFieldLiteralColumnIndexSupplierTest extends InitializedNullHandlingTest { + BitmapSerdeFactory roaringFactory = new RoaringBitmapSerdeFactory(null); + BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory( + roaringFactory.getBitmapFactory() + ); + GenericIndexed globalStrings; + FixedIndexed globalLongs; + FixedIndexed globalDoubles; + + @Before + public void setup() throws IOException + { + ByteBuffer stringBuffer = ByteBuffer.allocate(1 << 12); + ByteBuffer longBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer doubleBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + + GenericIndexedWriter stringWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "strings", + GenericIndexed.STRING_STRATEGY + ); + stringWriter.open(); + stringWriter.write(null); + stringWriter.write("a"); + stringWriter.write("b"); + stringWriter.write("fo"); + stringWriter.write("foo"); + stringWriter.write("fooo"); + stringWriter.write("z"); + writeToBuffer(stringBuffer, stringWriter); + + FixedIndexedWriter longWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + TypeStrategies.LONG, + ByteOrder.nativeOrder(), + Long.BYTES, + true + ); + longWriter.open(); + longWriter.write(1L); + longWriter.write(2L); + longWriter.write(3L); + longWriter.write(5L); + longWriter.write(100L); + longWriter.write(300L); + longWriter.write(9000L); + writeToBuffer(longBuffer, longWriter); + + FixedIndexedWriter doubleWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + TypeStrategies.DOUBLE, + ByteOrder.nativeOrder(), + Double.BYTES, + true + ); + doubleWriter.open(); + doubleWriter.write(1.0); + doubleWriter.write(1.1); + doubleWriter.write(1.2); + doubleWriter.write(2.0); + doubleWriter.write(2.5); + doubleWriter.write(3.3); + doubleWriter.write(6.6); + doubleWriter.write(9.9); + writeToBuffer(doubleBuffer, doubleWriter); + + globalStrings = GenericIndexed.read(stringBuffer, GenericIndexed.STRING_STRATEGY); + globalLongs = FixedIndexed.read(longBuffer, TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES); + globalDoubles = FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE, ByteOrder.nativeOrder(), Double.BYTES); + } + @Test - public void testRangeValueSkipping() + public void testSingleTypeStringColumnValueSetIndex() throws IOException { - FixedIndexed localDictionary = EasyMock.createMock(FixedIndexed.class); - GenericIndexed stringDictionary = EasyMock.createMock(GenericIndexed.class); - FixedIndexed longDictionary = EasyMock.createMock(FixedIndexed.class); - FixedIndexed doubleDictionary = EasyMock.createMock(FixedIndexed.class); - GenericIndexed bitmaps = EasyMock.createMock(GenericIndexed.class); + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringSupplier(); - RoaringBitmapFactory bitmapFactory = new RoaringBitmapFactory(); - MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap(); - bitmap.add(1); - ImmutableBitmap immutableBitmap = bitmapFactory.makeImmutableBitmap(bitmap); - MutableBitmap bitmap2 = bitmapFactory.makeEmptyMutableBitmap(); - bitmap2.add(2); - ImmutableBitmap immutableBitmap2 = bitmapFactory.makeImmutableBitmap(bitmap2); + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); - EasyMock.expect(stringDictionary.size()).andReturn(10).times(3); - EasyMock.expect(longDictionary.size()).andReturn(0).times(1); - EasyMock.expect(stringDictionary.indexOf("fo")).andReturn(3).times(2); - EasyMock.expect(stringDictionary.indexOf("fooo")).andReturn(5).times(2); - EasyMock.expect(stringDictionary.get(3)).andReturn("fo").times(1); - EasyMock.expect(stringDictionary.get(4)).andReturn("foo").times(1); - EasyMock.expect(stringDictionary.get(5)).andReturn("fooo").times(1); - EasyMock.expect(localDictionary.indexOf(3)).andReturn(0).times(2); - EasyMock.expect(localDictionary.indexOf(4)).andReturn(0).times(2); - EasyMock.expect(localDictionary.indexOf(5)).andReturn(1).times(2); - EasyMock.expect(localDictionary.indexOf(6)).andReturn(-2).times(2); - EasyMock.expect(bitmaps.get(0)).andReturn(immutableBitmap).times(1); - EasyMock.expect(bitmaps.get(1)).andReturn(immutableBitmap2).times(2); + // 10 rows + // local: [b, foo, fooo, z] + // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] - EasyMock.replay(localDictionary, stringDictionary, longDictionary, doubleDictionary, bitmaps); + BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 7, 8); - NestedFieldLiteralColumnIndexSupplier indexSupplier = new NestedFieldLiteralColumnIndexSupplier( - new NestedLiteralTypeInfo.TypeSet(new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue()), - bitmapFactory, - bitmaps, - localDictionary, - stringDictionary, - longDictionary, - doubleDictionary - ); + // non-existent in local column + columnIndex = valueSetIndex.forValue("fo"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "fooo", "z"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.8, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 2, 3, 4, 5, 6, 7, 8); + } + + @Test + public void testSingleTypeStringColumnRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringSupplier(); LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class); + Assert.assertNotNull(rangeIndex); - BitmapColumnIndex columnIndex = rangeIndex.forRange("fo", false, "fooo", false); - DefaultBitmapResultFactory defaultBitmapResultFactory = new DefaultBitmapResultFactory(bitmapFactory); - ImmutableBitmap result = columnIndex.computeBitmapResult(defaultBitmapResultFactory); - Assert.assertEquals(2, result.size()); - Assert.assertTrue(result.get(1)); - Assert.assertTrue(result.get(2)); + // 10 rows + // local: [b, foo, fooo, z] + // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] - // predicate skips first index - columnIndex = rangeIndex.forRange("fo", false, "fooo", false, "fooo"::equals); - result = columnIndex.computeBitmapResult(defaultBitmapResultFactory); - Assert.assertEquals(1, result.size()); - Assert.assertTrue(result.get(2)); - EasyMock.verify(localDictionary, stringDictionary, longDictionary, doubleDictionary, bitmaps); + BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 5, 9); + + forRange = rangeIndex.forRange(null, false, "g", true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); + + forRange = rangeIndex.forRange("f", false, null, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); + + forRange = rangeIndex.forRange("b", true, "fooo", true); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 9); + + forRange = rangeIndex.forRange("b", true, "fooo", false); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 5, 9); + + forRange = rangeIndex.forRange(null, true, "fooo", true); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 3, 7, 8, 9); + + forRange = rangeIndex.forRange("b", true, null, false); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); + + forRange = rangeIndex.forRange("b", false, null, true); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(null, true, "fooo", false); + Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + @Test + public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringSupplier(); + + LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [b, foo, fooo, z] + // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] + + BitmapColumnIndex forRange = rangeIndex.forRange( + "f", + true, + "g", + true, + s -> !"fooo".equals(s) + ); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 9); + + forRange = rangeIndex.forRange( + "f", + true, + "g", + true, + s -> "fooo".equals(s) + ); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 5); + + forRange = rangeIndex.forRange( + null, + false, + "z", + false, + s -> !"fooo".equals(s) + ); + Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 8, 9); + + forRange = rangeIndex.forRange( + null, + false, + "z", + true, + s -> !"fooo".equals(s) + ); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 3, 7, 8, 9); + + forRange = rangeIndex.forRange( + "f", + true, + null, + true, + s -> true + ); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); + } + + @Test + public void testSingleTypeStringColumnPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringSupplier(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("b", "z")) + ); + + // 10 rows + // local: [b, foo, fooo, z] + // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 4, 6, 7, 8); + } + + @Test + public void testSingleTypeStringColumnWithNullValueIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringWithNullsSupplier(); + + NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class); + Assert.assertNotNull(nullIndex); + + // 10 rows + // local: [null, b, foo, fooo, z] + // column: [foo, null, fooo, b, z, fooo, z, null, null, foo] + + BitmapColumnIndex columnIndex = nullIndex.forNull(); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 7, 8); + } + + @Test + public void testSingleTypeStringColumnWithNullValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringWithNullsSupplier(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [null, b, foo, fooo, z] + // column: [foo, null, fooo, b, z, fooo, z, null, null, foo] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 3); + + // non-existent in local column + columnIndex = valueSetIndex.forValue("fo"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "fooo", "z"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 3, 4, 5, 6); + } + + @Test + public void testSingleValueStringWithNullRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringWithNullsSupplier(); + + LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [null, b, foo, fooo, z] + // column: [foo, null, fooo, b, z, fooo, z, null, null, foo] + + BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 5, 9); + + forRange = rangeIndex.forRange(null, false, "g", true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 3, 5, 9); + + forRange = rangeIndex.forRange("f", false, null, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); + + forRange = rangeIndex.forRange("b", true, "fooo", true); + Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 9); + + forRange = rangeIndex.forRange("b", true, "fooo", false); + Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 5, 9); + + forRange = rangeIndex.forRange(null, true, "fooo", true); + Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 3, 9); + + forRange = rangeIndex.forRange("b", true, null, false); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); + + forRange = rangeIndex.forRange("b", false, null, true); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); + + forRange = rangeIndex.forRange(null, true, "fooo", false); + Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 3, 5, 9); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); + } + + @Test + public void testSingleValueStringWithNullPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeStringWithNullsSupplier(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("b", "z")) + ); + + // 10 rows + // local: [null, b, foo, fooo, z] + // column: [foo, null, fooo, b, z, fooo, z, null, null, foo] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 3, 4, 6); + } + + @Test + public void testSingleTypeLongColumnValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplier(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [1, 3, 100, 300] + // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("1"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 9); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1", "300", "700"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 2, 3, 7, 8, 9); + } + + @Test + public void testSingleTypeLongColumnRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplier(); + + NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [1, 3, 100, 300] + // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1] + + BitmapColumnIndex forRange = rangeIndex.forRange(10L, true, 400L, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 6, 7, 8); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + @Test + public void testSingleTypeLongColumnPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplier(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("1", "3")) + ); + + // 10 rows + // local: [1, 3, 100, 300] + // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 4, 5, 9); + } + + @Test + public void testSingleTypeLongColumnWithNullValueIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplierWithNull(); + + NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class); + Assert.assertNotNull(nullIndex); + + // 10 rows + // local: [null, 1, 3, 100, 300] + // column: [100, 1, null, 1, 3, null, 100, 300, null, 1] + + BitmapColumnIndex columnIndex = nullIndex.forNull(); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 5, 8); + } + + @Test + public void testSingleTypeLongColumnWithNullValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplierWithNull(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [null, 1, 3, 100, 300] + // column: [100, 1, null, 1, 3, null, 100, 300, null, 1] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("3"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 4); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1", "3", "300"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 4, 7, 9); + } + + @Test + public void testSingleValueLongWithNullRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplierWithNull(); + + NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [null, 1, 3, 100, 300] + // column: [100, 1, null, 1, 3, null, 100, 300, null, 1] + + BitmapColumnIndex forRange = rangeIndex.forRange(100, false, 700, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 6, 7); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9); + } + + @Test + public void testSingleValueLongWithNullPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeLongSupplierWithNull(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("3", "100")) + ); + + // 10 rows + // local: [null, 1, 3, 100, 300] + // column: [100, 1, null, 1, 3, null, 100, 300, null, 1] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 4, 6); + } + + @Test + public void testSingleTypeDoubleColumnValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplier(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [1.1, 1.2, 3.3, 6.6] + // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("1.2"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 4, 7); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1.2", "3.3", "6.6"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.7, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 3, 4, 5, 6, 7, 9); + } + + @Test + public void testSingleTypeDoubleColumnRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplier(); + + NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [1.1, 1.2, 3.3, 6.6] + // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3] + + BitmapColumnIndex forRange = rangeIndex.forRange(1.0, true, 5.0, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0); + + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(1.1, false, 3.3, false); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0); + + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(1.1, true, 3.3, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 3, 4, 6, 7, 9); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + @Test + public void testSingleTypeDoubleColumnPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplier(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("1.2", "3.3", "5.0")) + ); + + // 10 rows + // local: [1.1, 1.2, 3.3, 6.6] + // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 3, 4, 6, 7, 9); + } + + @Test + public void testSingleTypeDoubleColumnWithNullValueIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplierWithNull(); + + NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class); + Assert.assertNotNull(nullIndex); + + // 10 rows + // local: [null, 1.1, 1.2, 3.3, 6.6] + // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3] + + BitmapColumnIndex columnIndex = nullIndex.forNull(); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 6); + } + + @Test + public void testSingleTypeDoubleColumnWithNullValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplierWithNull(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [null, 1.1, 1.2, 3.3, 6.6] + // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("6.6"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 5); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1.2", "3.3", "7.7"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 4, 7, 9); + } + + @Test + public void testSingleValueDoubleWithNullRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplierWithNull(); + + NumericRangeIndex rangeIndex = indexSupplier.as(NumericRangeIndex.class); + Assert.assertNotNull(rangeIndex); + + // 10 rows + // local: [null, 1.1, 1.2, 3.3, 6.6] + // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3] + + BitmapColumnIndex forRange = rangeIndex.forRange(1.1, false, 5.0, true); + Assert.assertNotNull(forRange); + Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + + ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 7, 8, 9); + + forRange = rangeIndex.forRange(null, true, null, true); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9); + + forRange = rangeIndex.forRange(null, false, null, false); + Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + bitmap = forRange.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9); + } + + @Test + public void testSingleValueDoubleWithNullPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeSingleTypeDoubleSupplierWithNull(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("1.2", "3.3")) + ); + + // 10 rows + // local: [null, 1.1, 1.2, 3.3, 6.6] + // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 4, 7, 9); + } + + @Test + public void testVariantNullValueIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeVariantSupplierWithNull(); + + NullValueIndex nullIndex = indexSupplier.as(NullValueIndex.class); + Assert.assertNotNull(nullIndex); + + // 10 rows + // local: [null, b, z, 1, 300, 1.1, 9.9] + // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b] + + BitmapColumnIndex columnIndex = nullIndex.forNull(); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 2, 7); + } + + @Test + public void testVariantValueSetIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeVariantSupplierWithNull(); + + StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); + Assert.assertNotNull(valueSetIndex); + + // 10 rows + // local: [null, b, z, 1, 300, 1.1, 9.9] + // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b] + + BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 9); + + columnIndex = valueSetIndex.forValue("1"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 0, 5); + + columnIndex = valueSetIndex.forValue("1.1"); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 8); + + // set index + columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "300", "9.9", "1.6"))); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 4, 9); + } + + @Test + public void testVariantRangeIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeVariantSupplierWithNull(); + + LexicographicalRangeIndex rangeIndex = indexSupplier.as(LexicographicalRangeIndex.class); + Assert.assertNull(rangeIndex); + + NumericRangeIndex numericRangeIndex = indexSupplier.as(NumericRangeIndex.class); + Assert.assertNull(numericRangeIndex); + } + + @Test + public void testVariantPredicateIndex() throws IOException + { + NestedFieldLiteralColumnIndexSupplier indexSupplier = makeVariantSupplierWithNull(); + + DruidPredicateIndex predicateIndex = indexSupplier.as(DruidPredicateIndex.class); + Assert.assertNotNull(predicateIndex); + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("b", "z", "9.9", "300")) + ); + + // 10 rows + // local: [null, b, z, 1, 300, 1.1, 9.9] + // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b] + + BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); + Assert.assertNotNull(columnIndex); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); + checkBitmap(bitmap, 1, 3, 4, 6, 9); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringSupplier() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [b, foo, fooo, z] + // column: [foo, b, fooo, b, z, fooo, z, b, b, foo] + + // b + localDictionaryWriter.write(2); + bitmapWriter.write(fillBitmap(1, 3, 7, 8)); + + // foo + localDictionaryWriter.write(4); + bitmapWriter.write(fillBitmap(0, 9)); + + // fooo + localDictionaryWriter.write(5); + bitmapWriter.write(fillBitmap(2, 5)); + + // z + localDictionaryWriter.write(6); + bitmapWriter.write(fillBitmap(4, 6)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeStringWithNullsSupplier() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [null, b, foo, fooo, z] + // column: [foo, null, fooo, b, z, fooo, z, null, null, foo] + + // null + localDictionaryWriter.write(0); + bitmapWriter.write(fillBitmap(1, 7, 8)); + + // b + localDictionaryWriter.write(2); + bitmapWriter.write(fillBitmap(3)); + + // foo + localDictionaryWriter.write(4); + bitmapWriter.write(fillBitmap(0, 9)); + + // fooo + localDictionaryWriter.write(5); + bitmapWriter.write(fillBitmap(2, 5)); + + // z + localDictionaryWriter.write(6); + bitmapWriter.write(fillBitmap(4, 6)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeLongSupplier() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [1, 3, 100, 300] + // column: [100, 1, 300, 1, 3, 3, 100, 300, 300, 1] + + // 1 + localDictionaryWriter.write(7); + bitmapWriter.write(fillBitmap(1, 3, 9)); + + // 3 + localDictionaryWriter.write(9); + bitmapWriter.write(fillBitmap(4, 5)); + + // 100 + localDictionaryWriter.write(11); + bitmapWriter.write(fillBitmap(0, 6)); + + // 300 + localDictionaryWriter.write(12); + bitmapWriter.write(fillBitmap(2, 7, 8)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeLongSupplierWithNull() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [null, 1, 3, 100, 300] + // column: [100, 1, null, 1, 3, null, 100, 300, null, 1] + + // null + localDictionaryWriter.write(0); + bitmapWriter.write(fillBitmap(2, 5, 8)); + + // 1 + localDictionaryWriter.write(7); + bitmapWriter.write(fillBitmap(1, 3, 9)); + + // 3 + localDictionaryWriter.write(9); + bitmapWriter.write(fillBitmap(4)); + + // 100 + localDictionaryWriter.write(11); + bitmapWriter.write(fillBitmap(0, 6)); + + // 300 + localDictionaryWriter.write(12); + bitmapWriter.write(fillBitmap(7)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeDoubleSupplier() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [1.1, 1.2, 3.3, 6.6] + // column: [1.1, 1.1, 1.2, 3.3, 1.2, 6.6, 3.3, 1.2, 1.1, 3.3] + + // 1.1 + localDictionaryWriter.write(15); + bitmapWriter.write(fillBitmap(0, 1, 8)); + + // 1.2 + localDictionaryWriter.write(16); + bitmapWriter.write(fillBitmap(2, 4, 7)); + + // 3.3 + localDictionaryWriter.write(19); + bitmapWriter.write(fillBitmap(3, 6, 9)); + + // 6.6 + localDictionaryWriter.write(20); + bitmapWriter.write(fillBitmap(5)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeSingleTypeDoubleSupplierWithNull() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [null, 1.1, 1.2, 3.3, 6.6] + // column: [1.1, null, 1.2, null, 1.2, 6.6, null, 1.2, 1.1, 3.3] + + // null + localDictionaryWriter.write(0); + bitmapWriter.write(fillBitmap(1, 3, 6)); + + // 1.1 + localDictionaryWriter.write(15); + bitmapWriter.write(fillBitmap(0, 8)); + + // 1.2 + localDictionaryWriter.write(16); + bitmapWriter.write(fillBitmap(2, 4, 7)); + + // 3.3 + localDictionaryWriter.write(19); + bitmapWriter.write(fillBitmap(9)); + + // 6.6 + localDictionaryWriter.write(20); + bitmapWriter.write(fillBitmap(5)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private NestedFieldLiteralColumnIndexSupplier makeVariantSupplierWithNull() throws IOException + { + ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); + ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); + + FixedIndexedWriter localDictionaryWriter = new FixedIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES, + true + ); + localDictionaryWriter.open(); + GenericIndexedWriter bitmapWriter = new GenericIndexedWriter<>( + new OnHeapMemorySegmentWriteOutMedium(), + "bitmaps", + roaringFactory.getObjectStrategy() + ); + bitmapWriter.setObjectsNotSorted(); + bitmapWriter.open(); + + // 10 rows + // globals: [ + // [null, a, b, fo, foo, fooo, z], + // [1, 2, 3, 5, 100, 300, 9000], + // [1.0, 1.1, 1.2, 2.0, 2.5, 3.3, 6.6, 9.9] + // ] + // local: [null, b, z, 1, 300, 1.1, 9.9] + // column: [1, b, null, 9.9, 300, 1, z, null, 1.1, b] + + // null + localDictionaryWriter.write(0); + bitmapWriter.write(fillBitmap(2, 7)); + + // b + localDictionaryWriter.write(2); + bitmapWriter.write(fillBitmap(1, 9)); + + // z + localDictionaryWriter.write(6); + bitmapWriter.write(fillBitmap(6)); + + // 1 + localDictionaryWriter.write(7); + bitmapWriter.write(fillBitmap(0, 5)); + + // 300 + localDictionaryWriter.write(12); + bitmapWriter.write(fillBitmap(4)); + + // 1.1 + localDictionaryWriter.write(15); + bitmapWriter.write(fillBitmap(8)); + + // 9.9 + localDictionaryWriter.write(21); + bitmapWriter.write(fillBitmap(3)); + + writeToBuffer(localDictionaryBuffer, localDictionaryWriter); + writeToBuffer(bitmapsBuffer, bitmapWriter); + + FixedIndexed dictionary = FixedIndexed.read( + localDictionaryBuffer, + NestedDataColumnSerializer.INT_TYPE_STRATEGY, + ByteOrder.nativeOrder(), + Integer.BYTES + ); + + GenericIndexed bitmaps = GenericIndexed.read(bitmapsBuffer, roaringFactory.getObjectStrategy()); + + return new NestedFieldLiteralColumnIndexSupplier( + new NestedLiteralTypeInfo.TypeSet( + new NestedLiteralTypeInfo.MutableTypeSet().add(ColumnType.STRING) + .add(ColumnType.LONG) + .add(ColumnType.DOUBLE) + .getByteValue() + ), + roaringFactory.getBitmapFactory(), + bitmaps, + dictionary, + globalStrings, + globalLongs, + globalDoubles + ); + } + + private ImmutableBitmap fillBitmap(int... rows) + { + MutableBitmap bitmap = roaringFactory.getBitmapFactory().makeEmptyMutableBitmap(); + for (int i : rows) { + bitmap.add(i); + } + return roaringFactory.getBitmapFactory().makeImmutableBitmap(bitmap); + } + + void checkBitmap(ImmutableBitmap bitmap, int... expectedRows) + { + IntIterator iterator = bitmap.iterator(); + for (int i : expectedRows) { + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(i, iterator.next()); + } + Assert.assertFalse(iterator.hasNext()); + } + + static void writeToBuffer(ByteBuffer buffer, Serializer serializer) throws IOException + { + WritableByteChannel channel = new WritableByteChannel() + { + @Override + public int write(ByteBuffer src) + { + int size = src.remaining(); + buffer.put(src); + return size; + } + + @Override + public boolean isOpen() + { + return true; + } + + @Override + public void close() + { + } + }; + + serializer.writeTo(channel, null); + buffer.position(0); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index ea3c66d11da..182ef9ab950 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -852,7 +852,6 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .build() ), ImmutableList.of( - new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"100", 2L} ) ); @@ -954,6 +953,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .build() ), ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"100", 2L} ) ); @@ -1052,7 +1052,6 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .build() ), ImmutableList.of( - new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"2.02", 2L} ) ); @@ -1154,6 +1153,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .build() ), ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"2.02", 2L} ) );