diff --git a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java index 84b35cedc23..e74050f7417 100644 --- a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java +++ b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java @@ -187,7 +187,7 @@ public abstract class DictionaryEncodedColumnMerger> imp dimConversions.set(i, dictionaryMergeIterator.conversions[i]); } } - cardinality = dictionaryMergeIterator.counter; + cardinality = dictionaryMergeIterator.getCardinality(); } else if (numMergeIndex == 1) { writeDictionary(dimValueLookup); cardinality = dimValueLookup.size(); diff --git a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java index 56950d87808..2e570c7d3de 100644 --- a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java +++ b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java @@ -124,6 +124,11 @@ public class DictionaryMergingIterator> implements Close return value; } + public int getCardinality() + { + return counter; + } + protected PeekingIterator transformIndexedIterator(Indexed indexed) { return Iterators.peekingIterator( diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java index 70f67bc13ee..32fdbc607f9 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java @@ -42,6 +42,11 @@ public class ColumnBuilder @Nullable private SmooshedFileMapper fileMapper = null; + @SuppressWarnings("unused") + public ColumnCapabilitiesImpl getCapabilitiesBuilder() + { + return capabilitiesBuilder; + } public ColumnBuilder setFileMapper(SmooshedFileMapper fileMapper) { diff --git a/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java b/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java index 478983f9417..96550329a84 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java @@ -190,7 +190,7 @@ public enum CompressionStrategy *

* If the allocated buffer is a direct buffer, it should be registered to be freed with the given Closer. */ - ByteBuffer allocateInBuffer(int inputSize, Closer closer) + public ByteBuffer allocateInBuffer(int inputSize, Closer closer) { return ByteBuffer.allocate(inputSize); } @@ -203,7 +203,7 @@ public enum CompressionStrategy *

* If the allocated buffer is a direct buffer, it should be registered to be freed with the given Closer. */ - abstract ByteBuffer allocateOutBuffer(int inputSize, Closer closer); + public abstract ByteBuffer allocateOutBuffer(int inputSize, Closer closer); /** * Returns a ByteBuffer with compressed contents of in between it's position and limit. It may be the provided out @@ -221,7 +221,7 @@ public enum CompressionStrategy private static final UncompressedCompressor DEFAULT_COMPRESSOR = new UncompressedCompressor(); @Override - ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + public ByteBuffer allocateOutBuffer(int inputSize, Closer closer) { return ByteBuffer.allocate(inputSize); } @@ -333,7 +333,7 @@ public enum CompressionStrategy } @Override - ByteBuffer allocateInBuffer(int inputSize, Closer closer) + public ByteBuffer allocateInBuffer(int inputSize, Closer closer) { ByteBuffer inBuffer = ByteBuffer.allocateDirect(inputSize); closer.register(() -> ByteBufferUtils.free(inBuffer)); @@ -341,7 +341,7 @@ public enum CompressionStrategy } @Override - ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + public ByteBuffer allocateOutBuffer(int inputSize, Closer closer) { ByteBuffer outBuffer = ByteBuffer.allocateDirect(LZ4_HIGH.maxCompressedLength(inputSize)); closer.register(() -> ByteBufferUtils.free(outBuffer)); @@ -365,7 +365,7 @@ public enum CompressionStrategy private static final ZstdCompressor DEFAULT_COMPRESSOR = new ZstdCompressor(); @Override - ByteBuffer allocateInBuffer(int inputSize, Closer closer) + public ByteBuffer allocateInBuffer(int inputSize, Closer closer) { ByteBuffer inBuffer = ByteBuffer.allocateDirect(inputSize); closer.register(() -> ByteBufferUtils.free(inBuffer)); @@ -373,7 +373,7 @@ public enum CompressionStrategy } @Override - ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + public ByteBuffer allocateOutBuffer(int inputSize, Closer closer) { ByteBuffer outBuffer = ByteBuffer.allocateDirect((int) Zstd.compressBound(inputSize)); closer.register(() -> ByteBufferUtils.free(outBuffer)); diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java index 1f2be9ed1ec..44f6bebd02e 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java @@ -45,25 +45,6 @@ public class IncrementalIndexAdapter implements IndexableAdapter private final IncrementalIndex index; private final Map accessors; - private static class DimensionAccessor - { - private final IncrementalIndex.DimensionDesc dimensionDesc; - @Nullable - private final MutableBitmap[] invertedIndexes; - private final DimensionIndexer indexer; - - public DimensionAccessor(IncrementalIndex.DimensionDesc dimensionDesc) - { - this.dimensionDesc = dimensionDesc; - this.indexer = dimensionDesc.getIndexer(); - if (dimensionDesc.getCapabilities().hasBitmapIndexes()) { - this.invertedIndexes = new MutableBitmap[indexer.getCardinality() + 1]; - } else { - this.invertedIndexes = null; - } - } - } - public IncrementalIndexAdapter(Interval dataInterval, IncrementalIndex index, BitmapFactory bitmapFactory) { this.dataInterval = dataInterval; @@ -118,6 +99,12 @@ public class IncrementalIndexAdapter implements IndexableAdapter } } + @SuppressWarnings("unused") + public IncrementalIndex getIncrementalIndex() + { + return index; + } + @Override public Interval getDataInterval() { @@ -190,6 +177,24 @@ public class IncrementalIndexAdapter implements IndexableAdapter return new MutableBitmapValues(bitmapIndex); } + @Override + public String getMetricType(String metric) + { + return index.getMetricType(metric); + } + + @Override + public ColumnCapabilities getCapabilities(String column) + { + return index.getColumnCapabilities(column); + } + + @Override + public Metadata getMetadata() + { + return index.getMetadata(); + } + static class MutableBitmapValues implements BitmapValues { private final MutableBitmap bitmapIndex; @@ -212,21 +217,22 @@ public class IncrementalIndexAdapter implements IndexableAdapter } } - @Override - public String getMetricType(String metric) + private static class DimensionAccessor { - return index.getMetricType(metric); - } + private final IncrementalIndex.DimensionDesc dimensionDesc; + @Nullable + private final MutableBitmap[] invertedIndexes; + private final DimensionIndexer indexer; - @Override - public ColumnCapabilities getCapabilities(String column) - { - return index.getColumnCapabilities(column); - } - - @Override - public Metadata getMetadata() - { - return index.getMetadata(); + public DimensionAccessor(IncrementalIndex.DimensionDesc dimensionDesc) + { + this.dimensionDesc = dimensionDesc; + this.indexer = dimensionDesc.getIndexer(); + if (dimensionDesc.getCapabilities().hasBitmapIndexes()) { + this.invertedIndexes = new MutableBitmap[indexer.getCardinality() + 1]; + } else { + this.invertedIndexes = null; + } + } } } diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index bf1e66df355..23583724261 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -56,10 +56,10 @@ import java.nio.channels.WritableByteChannel; public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde { - private static final int NO_FLAGS = 0; - private static final int STARTING_FLAGS = Feature.NO_BITMAP_INDEX.getMask(); + public static final int NO_FLAGS = 0; + public static final int STARTING_FLAGS = Feature.NO_BITMAP_INDEX.getMask(); - enum Feature + public enum Feature { MULTI_VALUE, MULTI_VALUE_V3, @@ -76,7 +76,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde } } - enum VERSION + public enum VERSION { UNCOMPRESSED_SINGLE_VALUE, // 0x0 UNCOMPRESSED_MULTI_VALUE, // 0x1 diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java index 5820c537eab..9e08749c0bd 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java @@ -30,7 +30,7 @@ import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; -class ExpressionVectorInputBinding implements Expr.VectorInputBinding +public class ExpressionVectorInputBinding implements Expr.VectorInputBinding { private final Map numeric; private final Map objects; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java index 37c31ec1323..595bc08f613 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java @@ -847,6 +847,11 @@ public class AggregationTestHelper implements Closeable return results; } + public IndexIO getIndexIO() + { + return indexIO; + } + @Override public void close() throws IOException { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java index f02db13c454..0a24da42c54 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java @@ -460,6 +460,13 @@ public class OperatorConversions return this; } + @SuppressWarnings("unused") + public OperatorBuilder operandTypeInference(SqlOperandTypeInference operandTypeInference) + { + this.operandTypeInference = operandTypeInference; + return this; + } + /** * Creates a {@link SqlFunction} from this builder. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java index 97e6f48782b..9062a32d0ba 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java @@ -116,9 +116,14 @@ public class CastOperatorConversion implements SqlOperatorConversion } else if (SqlTypeName.DATETIME_TYPES.contains(fromType) && SqlTypeName.CHAR_TYPES.contains(toType)) { return castDateTimeToChar(plannerContext, operandExpression, fromType, Calcites.getColumnTypeForRelDataType(rexNode.getType())); } else { - // Handle other casts. - final ExprType fromExprType = EXPRESSION_TYPES.get(fromType); - final ExprType toExprType = EXPRESSION_TYPES.get(toType); + // Handle other casts. If either type is ANY, use the other type instead. If both are ANY, this means nulls + // downstream, Druid will try its best + final ExprType fromExprType = SqlTypeName.ANY.equals(fromType) + ? EXPRESSION_TYPES.get(toType) + : EXPRESSION_TYPES.get(fromType); + final ExprType toExprType = SqlTypeName.ANY.equals(toType) + ? EXPRESSION_TYPES.get(fromType) + : EXPRESSION_TYPES.get(toType); if (fromExprType == null || toExprType == null) { // We have no runtime type for these SQL types. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java index 28e8b665040..5e3f14595ae 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java @@ -32,7 +32,6 @@ import org.apache.druid.math.expr.Parser; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.Expressions; -import org.apache.druid.sql.calcite.table.RowSignatures; import java.math.BigDecimal; import java.util.Arrays; @@ -168,7 +167,7 @@ public class DruidRexExecutor implements RexExecutor } else { literal = rexBuilder.makeLiteral(Arrays.asList(exprResult.asArray()), constExp.getType(), true); } - } else if (sqlTypeName == SqlTypeName.OTHER && constExp.getType() instanceof RowSignatures.ComplexSqlType) { + } else if (sqlTypeName == SqlTypeName.OTHER) { // complex constant is not reducible, so just leave it as an expression literal = constExp; } else { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java index ec6f8c7ee30..b3464b9bd63 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.rel; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.ColumnType; @@ -127,6 +128,13 @@ public class VirtualColumnRegistry RelDataType typeHint ) { + if (typeHint.getSqlTypeName() == SqlTypeName.OTHER && expression.getDruidType() != null) { + // fall back to druid type if sql type isn't very helpful + return getOrCreateVirtualColumnForExpression( + expression, + expression.getDruidType() + ); + } return getOrCreateVirtualColumnForExpression( expression, Calcites.getColumnTypeForRelDataType(typeHint)