Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use object[] instead of string[] for vector expressions to be consistent with vector object selectors (#13209) 

* use object[] instead of string[] for vector expressions to be consistent with vector object selectors

* simplify
This commit is contained in:
Clint Wylie 2022-10-12 02:53:43 -07:00 committed by GitHub
parent 80e10ffe22
commit 59e2afc566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 278 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java
View File

@ -23,12 +23,11 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.vector.ExprEvalDoubleVector;
import org.apache.druid.math.expr.vector.ExprEvalLongVector;
import org.apache.druid.math.expr.vector.ExprEvalStringVector;
import org.apache.druid.math.expr.vector.ExprEvalObjectVector;
import org.apache.druid.math.expr.vector.ExprEvalVector;
import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Objects;
/**
@ -158,15 +157,12 @@ class IdentifierExpr implements Expr
if (inputType == null) {
// nil column, we can be anything, so be a string because it's the most flexible
// (numbers will be populated with default values in default mode and non-null)
return new IdentifierVectorProcessor<String[]>(ExpressionType.STRING)
return new IdentifierVectorProcessor<Object[]>(ExpressionType.STRING)
{
@Override
public ExprEvalVector<String[]> evalVector(VectorInputBinding bindings)
public ExprEvalVector<Object[]> evalVector(VectorInputBinding bindings)
{
// need to cast to string[] because null columns come out as object[]
return new ExprEvalStringVector(
Arrays.stream(bindings.getObjectVector(binding)).map(x -> (String) x).toArray(String[]::new)
);
return new ExprEvalObjectVector(bindings.getObjectVector(binding));
}
};
}
@ -190,12 +186,12 @@ class IdentifierExpr implements Expr
}
};
case STRING:
return new IdentifierVectorProcessor<String[]>(inputType)
return new IdentifierVectorProcessor<Object[]>(inputType)
{
@Override
public ExprEvalVector<String[]> evalVector(VectorInputBinding bindings)
public ExprEvalVector<Object[]> evalVector(VectorInputBinding bindings)
{
return new ExprEvalStringVector(bindings.getObjectVector(binding));
return new ExprEvalObjectVector(bindings.getObjectVector(binding));
}
};
default:

8
core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java
View File

@ -23,7 +23,7 @@ import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExpressionType;
public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor<String[]>
public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor<Object[]>
{
public CastToStringVectorProcessor(ExprVectorProcessor<?> delegate)
{
@ -31,15 +31,15 @@ public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor
}
@Override
public ExprEvalVector<String[]> evalVector(Expr.VectorInputBinding bindings)
public ExprEvalVector<Object[]> evalVector(Expr.VectorInputBinding bindings)
{
ExprEvalVector<?> result = delegate.evalVector(bindings);
final Object[] objects = result.getObjectVector();
final String[] output = new String[objects.length];
final Object[] output = new String[objects.length];
for (int i = 0; i < objects.length; i++) {
output[i] = Evals.asString(objects[i]);
}
return new ExprEvalStringVector(output);
return new ExprEvalObjectVector(output);
}
@Override

7
core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java → core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java
View File

@ -20,12 +20,13 @@
package org.apache.druid.math.expr.vector;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExpressionType;
import javax.annotation.Nullable;
public final class ExprEvalStringVector extends ExprEvalVector<String[]>
public final class ExprEvalObjectVector extends ExprEvalVector<Object[]>
{
@Nullable
private long[] longs;
@ -35,7 +36,7 @@ public final class ExprEvalStringVector extends ExprEvalVector<String[]>
@Nullable
private boolean[] numericNulls;
public ExprEvalStringVector(String[] values)
public ExprEvalObjectVector(Object[] values)
{
super(values, null);
}
@ -47,7 +48,7 @@ public final class ExprEvalStringVector extends ExprEvalVector<String[]>
doubles = new double[values.length];
numericNulls = new boolean[values.length];
for (int i = 0; i < values.length; i++) {
Number n = ExprEval.computeNumber(values[i]);
Number n = ExprEval.computeNumber(Evals.asString(values[i]));
if (n != null) {
longs[i] = n.longValue();
doubles[i] = n.doubleValue();

2
core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java
View File

@ -29,7 +29,7 @@ import java.lang.reflect.Array;
* Result of {@link ExprVectorProcessor#evalVector} which wraps the actual evaluated results of the operation over the
* input vector(s). Methods to get actual results mirror vectorized value and object selectors.
*
* The generic parameter T should be the native java array type of the vector result (long[], String[], etc.)
* The generic parameter T should be the native java array type of the vector result (long[], Object[], etc.)
*/
public abstract class ExprEvalVector<T>
{

17
core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java → core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java
View File

@ -22,14 +22,21 @@ package org.apache.druid.math.expr.vector;
import org.apache.druid.math.expr.ExpressionType;
/**
* specialized {@link UnivariateFunctionVectorObjectProcessor} for processing (String[]) -> long[]
* specialized {@link UnivariateFunctionVectorObjectProcessor} for processing (Object[]) -> long[]
*/
public abstract class LongOutStringInFunctionVectorProcessor
extends UnivariateFunctionVectorObjectProcessor<String[], long[]>
public abstract class LongOutObjectInFunctionVectorProcessor
extends UnivariateFunctionVectorObjectProcessor<Object[], long[]>
{
public LongOutStringInFunctionVectorProcessor(ExprVectorProcessor<String[]> processor, int maxVectorSize)
final ExpressionType inputType;
public LongOutObjectInFunctionVectorProcessor(
ExprVectorProcessor<Object[]> processor,
int maxVectorSize,
ExpressionType inputType
)
{
super(CastToTypeVectorProcessor.cast(processor, ExpressionType.STRING), maxVectorSize, new long[maxVectorSize]);
super(CastToTypeVectorProcessor.cast(processor, inputType), maxVectorSize, new long[maxVectorSize]);
this.inputType = inputType;
}
@Override

22
core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringsInFunctionVectorProcessor.java → core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java
View File

@ -23,19 +23,21 @@ import org.apache.druid.math.expr.ExpressionType;
import javax.annotation.Nullable;
public abstract class LongOutStringsInFunctionVectorProcessor extends BivariateFunctionVectorObjectProcessor<String[], String[], long[]>
public abstract class LongOutObjectsInFunctionVectorProcessor
extends BivariateFunctionVectorObjectProcessor<Object[], Object[], long[]>
{
private final boolean[] outNulls;
protected LongOutStringsInFunctionVectorProcessor(
ExprVectorProcessor<String[]> left,
ExprVectorProcessor<String[]> right,
int maxVectorSize
protected LongOutObjectsInFunctionVectorProcessor(
ExprVectorProcessor<Object[]> left,
ExprVectorProcessor<Object[]> right,
int maxVectorSize,
ExpressionType inputType
)
{
super(
CastToTypeVectorProcessor.cast(left, ExpressionType.STRING),
CastToTypeVectorProcessor.cast(right, ExpressionType.STRING),
CastToTypeVectorProcessor.cast(left, inputType),
CastToTypeVectorProcessor.cast(right, inputType),
maxVectorSize,
new long[maxVectorSize]
);
@ -43,12 +45,12 @@ public abstract class LongOutStringsInFunctionVectorProcessor extends BivariateF
}
@Nullable
abstract Long processValue(@Nullable String leftVal, @Nullable String rightVal);
abstract Long processValue(@Nullable Object leftVal, @Nullable Object rightVal);
@Override
void processIndex(String[] strings, String[] strings2, int i)
void processIndex(Object[] in1, Object[] in2, int i)
{
final Long outVal = processValue(strings[i], strings2[i]);
final Long outVal = processValue(in1[i], in2[i]);
if (outVal == null) {
outValues[i] = 0L;
outNulls[i] = true;

30
core/src/main/java/org/apache/druid/math/expr/vector/StringOutMultiStringInVectorProcessor.java → core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java
View File

@ -19,41 +19,43 @@
package org.apache.druid.math.expr.vector;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExpressionType;
/**
* many strings enter, one string leaves...
*/
public abstract class StringOutMultiStringInVectorProcessor implements ExprVectorProcessor<String[]>
public abstract class ObjectOutMultiObjectInVectorProcessor implements ExprVectorProcessor<Object[]>
{
final ExprVectorProcessor<String[]>[] inputs;
final ExprVectorProcessor<Object[]>[] inputs;
final int maxVectorSize;
final String[] outValues;
final boolean sqlCompatible = NullHandling.sqlCompatible();
final Object[] outValues;
protected StringOutMultiStringInVectorProcessor(
ExprVectorProcessor<String[]>[] inputs,
int maxVectorSize
final ExpressionType expressionType;
protected ObjectOutMultiObjectInVectorProcessor(
ExprVectorProcessor<Object[]>[] inputs,
int maxVectorSize,
ExpressionType objectType
)
{
this.inputs = inputs;
this.maxVectorSize = maxVectorSize;
this.outValues = new String[maxVectorSize];
this.outValues = new Object[maxVectorSize];
this.expressionType = objectType;
}
@Override
public ExpressionType getOutputType()
{
return ExpressionType.STRING;
return expressionType;
}
@Override
public ExprEvalVector<String[]> evalVector(Expr.VectorInputBinding bindings)
public ExprEvalVector<Object[]> evalVector(Expr.VectorInputBinding bindings)
{
final int currentSize = bindings.getCurrentVectorSize();
final String[][] in = new String[inputs.length][];
final Object[][] in = new Object[inputs.length][];
for (int i = 0; i < inputs.length; i++) {
in[i] = inputs[i].evalVector(bindings).values();
}
@ -61,8 +63,8 @@ public abstract class StringOutMultiStringInVectorProcessor implements ExprVecto
for (int i = 0; i < currentSize; i++) {
processIndex(in, i);
}
return new ExprEvalStringVector(outValues);
return new ExprEvalObjectVector(outValues);
}
abstract void processIndex(String[][] in, int i);
abstract void processIndex(Object[][] in, int i);
}

34
core/src/main/java/org/apache/druid/math/expr/vector/StringOutStringsInFunctionVectorProcessor.java → core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java
View File

@ -23,30 +23,34 @@ import org.apache.druid.math.expr.ExpressionType;
import javax.annotation.Nullable;
public abstract class StringOutStringsInFunctionVectorProcessor
extends BivariateFunctionVectorObjectProcessor<String[], String[], String[]>
public abstract class ObjectOutObjectsInFunctionVectorProcessor
extends BivariateFunctionVectorObjectProcessor<Object[], Object[], Object[]>
{
protected StringOutStringsInFunctionVectorProcessor(
ExprVectorProcessor<String[]> left,
ExprVectorProcessor<String[]> right,
int maxVectorSize
final ExpressionType expressionType;
protected ObjectOutObjectsInFunctionVectorProcessor(
ExprVectorProcessor<Object[]> left,
ExprVectorProcessor<Object[]> right,
int maxVectorSize,
ExpressionType expressionType
)
{
super(
CastToTypeVectorProcessor.cast(left, ExpressionType.STRING),
CastToTypeVectorProcessor.cast(right, ExpressionType.STRING),
CastToTypeVectorProcessor.cast(left, expressionType),
CastToTypeVectorProcessor.cast(right, expressionType),
maxVectorSize,
new String[maxVectorSize]
new Object[maxVectorSize]
);
this.expressionType = expressionType;
}
@Nullable
protected abstract String processValue(@Nullable String leftVal, @Nullable String rightVal);
protected abstract Object processValue(@Nullable Object leftVal, @Nullable Object rightVal);
@Override
void processIndex(String[] strings, String[] strings2, int i)
void processIndex(Object[] in1, Object[] in2, int i)
{
outValues[i] = processValue(strings[i], strings2[i]);
outValues[i] = processValue(in1[i], in2[i]);
}
@Override
@ -56,14 +60,14 @@ public abstract class StringOutStringsInFunctionVectorProcessor
}
@Override
ExprEvalVector<String[]> asEval()
ExprEvalVector<Object[]> asEval()
{
return new ExprEvalStringVector(outValues);
return new ExprEvalObjectVector(outValues);
}
@Override
public ExpressionType getOutputType()
{
return ExpressionType.STRING;
return expressionType;
}
}

120
core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java
View File

@ -38,7 +38,7 @@ public class VectorComparisonProcessors
Expr.VectorInputBindingInspector inspector,
Expr left,
Expr right,
Supplier<LongOutStringsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutObjectsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutLongsInFunctionVectorValueProcessor> longOutLongsInProcessor,
Supplier<DoubleOutLongDoubleInFunctionVectorValueProcessor> doubleOutLongDoubleInProcessor,
Supplier<DoubleOutDoubleLongInFunctionVectorValueProcessor> doubleOutDoubleLongInProcessor,
@ -81,7 +81,7 @@ public class VectorComparisonProcessors
Expr.VectorInputBindingInspector inspector,
Expr left,
Expr right,
Supplier<LongOutStringsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutObjectsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutLongsInFunctionVectorValueProcessor> longOutLongsInProcessor,
Supplier<LongOutLongDoubleInFunctionVectorValueProcessor> longOutLongDoubleInProcessor,
Supplier<LongOutDoubleLongInFunctionVectorValueProcessor> longOutDoubleLongInProcessor,
@ -136,15 +136,16 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Objects.equals(leftVal, rightVal));
}
@ -203,15 +204,16 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Objects.equals(leftVal, rightVal));
}
@ -278,15 +280,16 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(!Objects.equals(leftVal, rightVal));
}
@ -345,15 +348,16 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(!Objects.equals(leftVal, rightVal));
}
@ -420,17 +424,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) >= 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -487,17 +494,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) >= 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -562,17 +572,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) > 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -629,17 +642,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) > 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -704,17 +720,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) <= 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -771,17 +790,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) <= 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -846,17 +868,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) < 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
@ -913,17 +938,20 @@ public class VectorComparisonProcessors
inspector,
left,
right,
() -> new LongOutStringsInFunctionVectorProcessor(
() -> new LongOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable String leftVal, @Nullable String rightVal)
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Comparators.<String>naturalNullsFirst().compare(leftVal, rightVal) < 0);
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(

90
core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java
View File

@ -82,9 +82,9 @@ public class VectorProcessors
public static <T> ExprVectorProcessor<T> constant(@Nullable String constant, int maxVectorSize)
{
final String[] strings = new String[maxVectorSize];
final Object[] strings = new Object[maxVectorSize];
Arrays.fill(strings, constant);
final ExprEvalStringVector eval = new ExprEvalStringVector(strings);
final ExprEvalObjectVector eval = new ExprEvalObjectVector(strings);
return new ExprVectorProcessor<T>()
{
@Override
@ -159,23 +159,29 @@ public class VectorProcessors
public static <T> ExprVectorProcessor<T> parseLong(Expr.VectorInputBindingInspector inspector, Expr arg, int radix)
{
final ExprVectorProcessor<?> processor = new LongOutStringInFunctionVectorProcessor(
CastToTypeVectorProcessor.cast(arg.buildVectorized(inspector), ExpressionType.STRING),
inspector.getMaxVectorSize()
final ExprVectorProcessor<?> processor = new LongOutObjectInFunctionVectorProcessor(
arg.buildVectorized(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Override
public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i)
public void processIndex(Object[] strings, long[] longs, boolean[] outputNulls, int i)
{
try {
final String input = strings[i];
if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) {
// Strip leading 0x from hex strings.
longs[i] = Long.parseLong(input.substring(2), radix);
final String input = (String) strings[i];
if (input == null) {
longs[i] = 0L;
outputNulls[i] = NullHandling.sqlCompatible();
} else {
longs[i] = Long.parseLong(input, radix);
if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) {
// Strip leading 0x from hex strings.
longs[i] = Long.parseLong(input.substring(2), radix);
} else {
longs[i] = Long.parseLong(input, radix);
}
outputNulls[i] = false;
}
outputNulls[i] = false;
}
catch (NumberFormatException e) {
longs[i] = 0L;
@ -199,16 +205,16 @@ public class VectorProcessors
ExprVectorProcessor<?> processor = null;
if (Types.is(type, ExprType.STRING)) {
final ExprVectorProcessor<String[]> input = expr.buildVectorized(inspector);
final ExprVectorProcessor<Object[]> input = expr.buildVectorized(inspector);
processor = new ExprVectorProcessor<long[]>()
{
@Override
public ExprEvalVector<long[]> evalVector(Expr.VectorInputBinding bindings)
{
final ExprEvalVector<String[]> inputEval = input.evalVector(bindings);
final ExprEvalVector<Object[]> inputEval = input.evalVector(bindings);
final int currentSize = bindings.getCurrentVectorSize();
final String[] values = inputEval.values();
final Object[] values = inputEval.values();
for (int i = 0; i < currentSize; i++) {
if (values[i] == null) {
outputValues[i] = 1L;
@ -307,16 +313,16 @@ public class VectorProcessors
ExprVectorProcessor<?> processor = null;
if (Types.is(type, ExprType.STRING)) {
final ExprVectorProcessor<String[]> input = expr.buildVectorized(inspector);
final ExprVectorProcessor<Object[]> input = expr.buildVectorized(inspector);
processor = new ExprVectorProcessor<long[]>()
{
@Override
public ExprEvalVector<long[]> evalVector(Expr.VectorInputBinding bindings)
{
final ExprEvalVector<String[]> inputEval = input.evalVector(bindings);
final ExprEvalVector<Object[]> inputEval = input.evalVector(bindings);
final int currentSize = bindings.getCurrentVectorSize();
final String[] values = inputEval.values();
final Object[] values = inputEval.values();
for (int i = 0; i < currentSize; i++) {
if (values[i] == null) {
outputValues[i] = 0L;
@ -483,19 +489,19 @@ public class VectorProcessors
return new ExprEvalDoubleVector(output, outputNulls);
}
},
() -> new SymmetricalBivariateFunctionVectorProcessor<String[]>(
() -> new SymmetricalBivariateFunctionVectorProcessor<Object[]>(
ExpressionType.STRING,
left.buildVectorized(inspector),
right.buildVectorized(inspector)
)
{
final String[] output = new String[maxVectorSize];
final Object[] output = new Object[maxVectorSize];
@Override
public void processIndex(
String[] leftInput,
Object[] leftInput,
@Nullable boolean[] leftNulls,
String[] rightInput,
Object[] rightInput,
@Nullable boolean[] rightNulls,
int i
)
@ -504,9 +510,9 @@ public class VectorProcessors
}
@Override
public ExprEvalVector<String[]> asEval()
public ExprEvalVector<Object[]> asEval()
{
return new ExprEvalStringVector(output);
return new ExprEvalObjectVector(output);
}
}
);
@ -518,14 +524,18 @@ public class VectorProcessors
final int maxVectorSize = inspector.getMaxVectorSize();
ExprVectorProcessor<?> processor = null;
if (Types.is(inputType, ExprType.STRING)) {
processor = new LongOutStringInFunctionVectorProcessor(expr.buildVectorized(inspector), maxVectorSize)
processor = new LongOutObjectInFunctionVectorProcessor(
expr.buildVectorized(inspector),
maxVectorSize,
ExpressionType.STRING
)
{
@Override
public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i)
public void processIndex(Object[] strings, long[] longs, boolean[] outputNulls, int i)
{
outputNulls[i] = strings[i] == null;
if (!outputNulls[i]) {
longs[i] = Evals.asLong(!Evals.asBoolean(strings[i]));
longs[i] = Evals.asLong(!Evals.asBoolean((String) strings[i]));
}
}
};
@ -670,7 +680,7 @@ public class VectorProcessors
return new ExprEvalLongVector(output, outputNulls);
}
},
() -> new BivariateFunctionVectorProcessor<String[], String[], long[]>(
() -> new BivariateFunctionVectorProcessor<Object[], Object[], long[]>(
ExpressionType.LONG,
left.buildVectorized(inspector),
right.buildVectorized(inspector)
@ -681,9 +691,9 @@ public class VectorProcessors
@Override
public void processIndex(
String[] leftInput,
Object[] leftInput,
@Nullable boolean[] leftNulls,
String[] rightInput,
Object[] rightInput,
@Nullable boolean[] rightNulls,
int i
)
@ -697,17 +707,17 @@ public class VectorProcessors
outputNulls[i] = true;
return;
}
final boolean bool = Evals.asBoolean(rightInput[i]);
final boolean bool = Evals.asBoolean((String) rightInput[i]);
output[i] = Evals.asLong(bool);
outputNulls[i] = !bool;
return;
} else if (rightNull) {
final boolean bool = Evals.asBoolean(leftInput[i]);
final boolean bool = Evals.asBoolean((String) leftInput[i]);
output[i] = Evals.asLong(bool);
outputNulls[i] = !bool;
return;
}
output[i] = Evals.asLong(Evals.asBoolean(leftInput[i]) || Evals.asBoolean(rightInput[i]));
output[i] = Evals.asLong(Evals.asBoolean((String) leftInput[i]) || Evals.asBoolean((String) rightInput[i]));
}
@Override
@ -824,7 +834,7 @@ public class VectorProcessors
return new ExprEvalLongVector(output, outputNulls);
}
},
() -> new BivariateFunctionVectorProcessor<String[], String[], long[]>(
() -> new BivariateFunctionVectorProcessor<Object[], Object[], long[]>(
ExpressionType.STRING,
left.buildVectorized(inputTypes),
right.buildVectorized(inputTypes)
@ -835,9 +845,9 @@ public class VectorProcessors
@Override
public void processIndex(
String[] leftInput,
Object[] leftInput,
@Nullable boolean[] leftNulls,
String[] rightInput,
Object[] rightInput,
@Nullable boolean[] rightNulls,
int i
)
@ -851,17 +861,19 @@ public class VectorProcessors
outputNulls[i] = true;
return;
}
final boolean bool = Evals.asBoolean(rightInput[i]);
final boolean bool = Evals.asBoolean((String) rightInput[i]);
output[i] = Evals.asLong(bool);
outputNulls[i] = bool;
return;
} else if (rightNull) {
final boolean bool = Evals.asBoolean(leftInput[i]);
final boolean bool = Evals.asBoolean((String) leftInput[i]);
output[i] = Evals.asLong(bool);
outputNulls[i] = bool;
return;
}
output[i] = Evals.asLong(Evals.asBoolean(leftInput[i]) && Evals.asBoolean(rightInput[i]));
output[i] = Evals.asLong(
Evals.asBoolean((String) leftInput[i]) && Evals.asBoolean((String) rightInput[i])
);
}
@Override

38
core/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.math.expr.vector;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExpressionType;
@ -32,32 +33,35 @@ public class VectorStringProcessors
{
final ExprVectorProcessor processor;
if (NullHandling.sqlCompatible()) {
processor = new StringOutStringsInFunctionVectorProcessor(
processor = new ObjectOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
protected String processValue(@Nullable String leftVal, @Nullable String rightVal)
protected String processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
// in sql compatible mode, nulls are handled by super class and never make it here...
return leftVal + rightVal;
return leftVal + (String) rightVal;
}
};
} else {
processor = new StringOutStringsInFunctionVectorProcessor(
processor = new ObjectOutObjectsInFunctionVectorProcessor(
left.buildVectorized(inspector),
right.buildVectorized(inspector),
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
protected String processValue(@Nullable String leftVal, @Nullable String rightVal)
protected Object processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return NullHandling.nullToEmptyIfNeeded(leftVal) + NullHandling.nullToEmptyIfNeeded(rightVal);
return NullHandling.nullToEmptyIfNeeded((String) leftVal)
+ NullHandling.nullToEmptyIfNeeded((String) rightVal);
}
};
}
@ -66,28 +70,32 @@ public class VectorStringProcessors
public static <T> ExprVectorProcessor<T> concat(Expr.VectorInputBindingInspector inspector, List<Expr> inputs)
{
final ExprVectorProcessor<String[]>[] inputProcessors = new ExprVectorProcessor[inputs.size()];
final ExprVectorProcessor<Object[]>[] inputProcessors = new ExprVectorProcessor[inputs.size()];
for (int i = 0; i < inputs.size(); i++) {
inputProcessors[i] = CastToTypeVectorProcessor.cast(inputs.get(i).buildVectorized(inspector), ExpressionType.STRING);
inputProcessors[i] = CastToTypeVectorProcessor.cast(
inputs.get(i).buildVectorized(inspector),
ExpressionType.STRING
);
}
final ExprVectorProcessor processor = new StringOutMultiStringInVectorProcessor(
final ExprVectorProcessor processor = new ObjectOutMultiObjectInVectorProcessor(
inputProcessors,
inspector.getMaxVectorSize()
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Override
void processIndex(String[][] in, int i)
void processIndex(Object[][] in, int i)
{
// Result of concatenation is null if any of the Values is null.
// e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec.
String first = NullHandling.nullToEmptyIfNeeded(in[0][i]);
String first = NullHandling.nullToEmptyIfNeeded(Evals.asString(in[0][i]));
if (first == null) {
outValues[i] = null;
return;
}
final StringBuilder builder = new StringBuilder(first);
for (int inputNumber = 1; inputNumber < in.length; inputNumber++) {
final String s = NullHandling.nullToEmptyIfNeeded(in[inputNumber][i]);
final String s = NullHandling.nullToEmptyIfNeeded(Evals.asString(in[inputNumber][i]));
if (s == null) {
outValues[i] = null;
return;

2
processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java
View File

@ -561,7 +561,7 @@ public class StringDictionaryEncodedColumn implements DictionaryEncodedColumn<St
class DictionaryEncodedStringSingleValueVectorObjectSelector implements VectorObjectSelector
{
private final int[] vector = new int[offset.getMaxVectorSize()];
private final String[] strings = new String[offset.getMaxVectorSize()];
private final Object[] strings = new Object[offset.getMaxVectorSize()];
private int id = ReadableVectorInspector.NULL_ID;
@Override

4
processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java
View File

@ -69,9 +69,9 @@ public class ExpressionVectorInputBinding implements Expr.VectorInputBinding
}
@Override
public <T> T[] getObjectVector(String name)
public Object[] getObjectVector(String name)
{
return (T[]) objects.getOrDefault(name, nilSelector).getObjectVector();
return objects.getOrDefault(name, nilSelector).getObjectVector();
}
@Override

7
processing/src/main/java/org/apache/druid/segment/virtual/SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.segment.virtual;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.vector.ExprVectorProcessor;
@ -43,7 +44,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto
implements SingleValueDimensionVectorSelector
{
private final SingleValueDimensionVectorSelector selector;
private final ExprVectorProcessor<String[]> stringProcessor;
private final ExprVectorProcessor<Object[]> stringProcessor;
private final StringLookupVectorInputBindings inputBinding;
public SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector(
@ -75,7 +76,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto
public String lookupName(int id)
{
inputBinding.currentValue[0] = selector.lookupName(id);
return stringProcessor.evalVector(inputBinding).values()[0];
return Evals.asString(stringProcessor.evalVector(inputBinding).values()[0]);
}
@Override
@ -118,7 +119,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto
*/
private static final class StringLookupVectorInputBindings implements Expr.VectorInputBinding
{
private final String[] currentValue = new String[1];
private final Object[] currentValue = new Object[1];
@Nullable
@Override

44
processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java
View File

@ -35,12 +35,14 @@ import org.apache.druid.query.aggregation.AggregationTestHelper;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.query.groupby.strategy.GroupByStrategySelector;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.virtual.NestedFieldVirtualColumn;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.After;
@ -76,6 +78,8 @@ public class NestedDataGroupByQueryTest extends InitializedNullHandlingTest
private final TrinaryFn<AggregationTestHelper, TemporaryFolder, Closer, List<Segment>> segmentsGenerator;
private final String segmentsName;
private boolean cannotVectorize = false;
public NestedDataGroupByQueryTest(
GroupByQueryConfig config,
TrinaryFn<AggregationTestHelper, TemporaryFolder, Closer, List<Segment>> segmentGenerator,
@ -273,6 +277,46 @@ public class NestedDataGroupByQueryTest extends InitializedNullHandlingTest
);
}
@Test
public void testGroupByNonExistentVirtualColumn()
{
if (GroupByStrategySelector.STRATEGY_V1.equals(config.getDefaultStrategy())) {
expectedException.expect(RuntimeException.class);
expectedException.expectMessage(
"GroupBy v1 does not support dimension selectors with unknown cardinality."
);
}
GroupByQuery groupQuery = GroupByQuery.builder()
.setDataSource("test_datasource")
.setGranularity(Granularities.ALL)
.setInterval(Intervals.ETERNITY)
.setDimensions(DefaultDimensionSpec.of("v1"))
.setVirtualColumns(
new NestedFieldVirtualColumn("fake", "$.fake", "v0", ColumnType.STRING),
new ExpressionVirtualColumn(
"v1",
"concat(v0, 'foo')",
ColumnType.STRING,
TestExprMacroTable.INSTANCE
)
)
.setAggregatorSpecs(new CountAggregatorFactory("count"))
.setContext(getContext())
.build();
Sequence<ResultRow> seq = helper.runQueryOnSegmentsObjs(segmentsGenerator.apply(helper, tempFolder, closer), groupQuery);
List<ResultRow> results = seq.toList();
verifyResults(
groupQuery.getResultRowSignature(),
results,
NullHandling.sqlCompatible()
? ImmutableList.of(new Object[]{null, 16L})
: ImmutableList.of(new Object[]{"foo", 16L})
);
}
private static void verifyResults(RowSignature rowSignature, List<ResultRow> results, List<Object[]> expected)
{
LOG.info("results:\n%s", results);

4
processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java
View File

@ -20,7 +20,7 @@
package org.apache.druid.segment.virtual;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.vector.ExprEvalStringVector;
import org.apache.druid.math.expr.vector.ExprEvalObjectVector;
import org.apache.druid.math.expr.vector.ExprEvalVector;
import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.easymock.EasyMock;
@ -57,7 +57,7 @@ public class ExpressionVectorObjectSelectorTest
public void testSelectObject()
{
final String[] vector = new String[]{"1", "2", null, "3"};
ExprEvalVector vectorEval = new ExprEvalStringVector(vector);
ExprEvalVector vectorEval = new ExprEvalObjectVector(vector);
EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes();
EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once();
EasyMock.replay(binding, vectorProcessor);

3
processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java
View File

@ -43,6 +43,7 @@ import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
import org.apache.druid.segment.vector.VectorCursor;
import org.apache.druid.segment.vector.VectorObjectSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.junit.AfterClass;
@ -60,7 +61,7 @@ import java.util.List;
import java.util.stream.Collectors;
@RunWith(Parameterized.class)
public class ExpressionVectorSelectorsTest
public class ExpressionVectorSelectorsTest extends InitializedNullHandlingTest
{
private static List<String> EXPRESSIONS = ImmutableList.of(
"long1 * long2",