Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize the reading of numerical frame arrays in MSQ (#15175)

This commit is contained in:
Laksh Singla 2023-10-18 02:33:42 +05:30 committed by GitHub
parent 953ce79439
commit b4540ed5d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 241 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java
View File

@ -37,6 +37,7 @@ public class DoubleArrayFieldReader extends NumericArrayFieldReader
{
return new NumericArrayFieldSelector<Double>(memory, fieldPointer)
{
private static final int FIELD_SIZE = Byte.BYTES + Double.BYTES;
final SettableFieldPointer fieldPointer = new SettableFieldPointer();
final ColumnValueSelector<?> columnValueSelector =
DoubleFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer);
@ -45,7 +46,7 @@ public class DoubleArrayFieldReader extends NumericArrayFieldReader
@Override
public Double getIndividualValueAtMemory(long position)
{
fieldPointer.setPosition(position);
fieldPointer.setPositionAndLength(position, FIELD_SIZE);
if (columnValueSelector.isNull()) {
return null;
}
@ -55,7 +56,7 @@ public class DoubleArrayFieldReader extends NumericArrayFieldReader
@Override
public int getIndividualFieldSize()
{
return Byte.BYTES + Double.BYTES;
return FIELD_SIZE;
}
};
}

5
processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java
View File

@ -37,6 +37,7 @@ public class FloatArrayFieldReader extends NumericArrayFieldReader
{
return new NumericArrayFieldSelector<Float>(memory, fieldPointer)
{
private static final int FIELD_SIZE = Byte.BYTES + Float.BYTES;
final SettableFieldPointer fieldPointer = new SettableFieldPointer();
final ColumnValueSelector<?> columnValueSelector =
FloatFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer);
@ -45,7 +46,7 @@ public class FloatArrayFieldReader extends NumericArrayFieldReader
@Override
public Float getIndividualValueAtMemory(long position)
{
fieldPointer.setPosition(position);
fieldPointer.setPositionAndLength(position, FIELD_SIZE);
if (columnValueSelector.isNull()) {
return null;
}
@ -55,7 +56,7 @@ public class FloatArrayFieldReader extends NumericArrayFieldReader
@Override
public int getIndividualFieldSize()
{
return Byte.BYTES + Float.BYTES;
return FIELD_SIZE;
}
};
}

5
processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java
View File

@ -37,6 +37,7 @@ public class LongArrayFieldReader extends NumericArrayFieldReader
{
return new NumericArrayFieldSelector<Long>(memory, fieldPointer)
{
private static final int FIELD_SIZE = Byte.BYTES + Long.BYTES;
final SettableFieldPointer fieldPointer = new SettableFieldPointer();
final ColumnValueSelector<?> columnValueSelector =
LongFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer);
@ -45,7 +46,7 @@ public class LongArrayFieldReader extends NumericArrayFieldReader
@Override
public Long getIndividualValueAtMemory(long position)
{
fieldPointer.setPosition(position);
fieldPointer.setPositionAndLength(position, FIELD_SIZE);
if (columnValueSelector.isNull()) {
return null;
}
@ -55,7 +56,7 @@ public class LongArrayFieldReader extends NumericArrayFieldReader
@Override
public int getIndividualFieldSize()
{
return Byte.BYTES + Long.BYTES;
return FIELD_SIZE;
}
};
}

57
processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java
View File

@ -25,8 +25,6 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnValueSelector;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
/**
* Base implementation of the column value selector that the concrete numeric field reader implementations inherit from.
@ -66,12 +64,8 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
/**
* Value of the row at the location beginning at {@link #currentFieldPosition}
*/
private final List<ElementType> currentRow = new ArrayList<>();
/**
* Nullity of the row at the location beginning at {@link #currentFieldPosition}
*/
private boolean currentRowIsNull;
@Nullable
private Number[] currentRow = null;
public NumericArrayFieldSelector(final Memory memory, final ReadableFieldPointer fieldPointer)
{
@ -89,13 +83,7 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
@Override
public Object getObject()
{
final List<ElementType> currentArray = computeCurrentArray();
if (currentArray == null) {
return null;
}
return currentArray.toArray();
return computeCurrentArray();
}
@Override
@ -143,34 +131,29 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
public abstract int getIndividualFieldSize();
@Nullable
private List<ElementType> computeCurrentArray()
private Number[] computeCurrentArray()
{
final long fieldPosition = fieldPointer.position();
final long fieldLength = fieldPointer.length();
if (fieldPosition != currentFieldPosition) {
updateCurrentArray(fieldPosition);
updateCurrentArray(fieldPosition, fieldLength);
}
this.currentFieldPosition = fieldPosition;
if (currentRowIsNull) {
return null;
}
return currentRow;
}
private void updateCurrentArray(final long fieldPosition)
private void updateCurrentArray(final long fieldPosition, final long fieldLength)
{
currentRow.clear();
currentRowIsNull = false;
currentRow = null;
long position = fieldPosition;
long limit = memory.getCapacity();
// Check the first byte, and if it is null, update the current value to null and return
if (isNull()) {
currentRowIsNull = true;
// Already set the currentRow to null
return;
}
@ -179,9 +162,13 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
position++;
}
int numElements = numElements(fieldLength);
currentRow = new Number[numElements];
// Sanity check, to make sure that we see the rowTerminator at the end
boolean rowTerminatorSeen = false;
int curElement = 0;
while (position < limit) {
final byte kind = memory.getByte(position);
@ -193,12 +180,26 @@ public abstract class NumericArrayFieldSelector<ElementType extends Number> impl
// If terminator not seen, then read the field at that location, and increment the position by the element's field
// size to read the next element.
currentRow.add(getIndividualValueAtMemory(position));
currentRow[curElement] = getIndividualValueAtMemory(position);
position += getIndividualFieldSize();
curElement++;
}
if (!rowTerminatorSeen) {
if (!rowTerminatorSeen || curElement != numElements) {
throw DruidException.defensive("Unexpected end of field");
}
}
int numElements(long fieldSize)
{
if (fieldSize <= 1) {
throw DruidException.defensive("fieldSize should be greater than 1 for non null array elements");
}
// Remove one byte for the nullity byte, and one for the array terminator
long cumulativeFieldSize = fieldSize - Byte.BYTES - Byte.BYTES;
if (cumulativeFieldSize % getIndividualFieldSize() != 0) {
throw DruidException.defensive("cumulativeFieldSize should be a multiple of the individual fieldSize");
}
return Math.toIntExact(cumulativeFieldSize / getIndividualFieldSize());
}
}

5
processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java
View File

@ -31,4 +31,9 @@ public interface ReadableFieldPointer
* Starting position of the field.
*/
long position();
/**
* Length of the field.
*/
long length();
}

71
processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java
View File

@ -24,6 +24,10 @@ import org.apache.druid.frame.segment.row.ReadableFrameRowPointer;
/**
* A {@link ReadableFieldPointer} that is derived from a row-based frame.
*
* Returns the position and the length of a field at a particular position for the row that the rowPointer is pointing
* to at the time. It caches the values of the position and the length based on position of the rowPointer.
* This method is not thread-safe
*/
public class RowMemoryFieldPointer implements ReadableFieldPointer
{
@ -32,6 +36,16 @@ public class RowMemoryFieldPointer implements ReadableFieldPointer
private final int fieldNumber;
private final int fieldCount;
// Caching of position() calls
private long rowWithCachedPosition = -1L;
private long cachedPosition = -1L;
// Caching of length() calls
// We cache the length() calls separately, because not all field types call length(), therefore it's wasteful to
// compute and cache length() if we are not reading it
private long rowWithCachedLength = -1L;
private long cachedLength = -1L;
public RowMemoryFieldPointer(
final Memory memory,
final ReadableFrameRowPointer rowPointer,
@ -47,6 +61,63 @@ public class RowMemoryFieldPointer implements ReadableFieldPointer
@Override
public long position()
{
updatePosition();
return cachedPosition;
}
@Override
public long length()
{
updatePositionAndLength();
return cachedLength;
}
private void updatePosition()
{
long rowPointerPosition = rowPointer.position();
if (rowPointerPosition == rowWithCachedPosition) {
return;
}
// Update the cached position for position()
rowWithCachedPosition = rowPointerPosition;
// Update the start position
cachedPosition = startPosition(fieldNumber);
}
// Not all field types call length(), and therefore there's no need to cache the length of the field. This method
// updates both the position and the length.
private void updatePositionAndLength()
{
updatePosition();
// rowPointerPosition = rowPointer.position() = rowWithCachedPosition, since that was updated in the call to update
// position above
long rowPointerPosition = rowWithCachedPosition;
if (rowPointerPosition == rowWithCachedLength) {
return;
}
// Update the cached position for length()
rowWithCachedLength = rowPointerPosition;
if (fieldNumber == fieldCount - 1) {
// If the field is the last field in the row, then the length of the field would be the end of the row minus the
// start position of the field. End of the row is the start of the row plus the length of the row.
cachedLength = (rowPointerPosition + rowPointer.length()) - cachedPosition;
} else {
// Else the length of the field would be the difference between the start position of the given field and
// the subsequent field
cachedLength = startPosition(fieldNumber + 1) - cachedPosition;
}
}
/**
* Given a fieldNumber, computes the start position of the field. Requires a memory access to read the start position,
* therefore callers should cache the value for better efficiency.
*/
private long startPosition(int fieldNumber)
{
if (fieldNumber == 0) {
// First field starts after the field end pointers -- one integer per field.

16
processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java
View File

@ -20,16 +20,20 @@
package org.apache.druid.frame.field;
/**
* A simple {@link ReadableFieldPointer} that returns the position that was set on its object.
* A simple {@link ReadableFieldPointer} that returns the position and the length that was set on its object.
*/
public class SettableFieldPointer implements ReadableFieldPointer
{
long position = 0;
long length = -1;
public void setPosition(long position)
/**
* Sets the position and the length to be returned when interface's methods are called.
*/
public void setPositionAndLength(long position, long length)
{
this.position = position;
this.length = length;
}
@Override
@ -37,4 +41,10 @@ public class SettableFieldPointer implements ReadableFieldPointer
{
return position;
}
@Override
public long length()
{
return length;
}
}

6
processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java
View File

@ -30,7 +30,13 @@ import org.apache.druid.frame.write.RowBasedFrameWriter;
*/
public interface ReadableFrameRowPointer
{
/**
* Position of the start of the row relative to the start of the Frame
*/
long position();
/**
* Length of the row (in bytes)
*/
long length();
}

4
processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java
View File

@ -123,7 +123,7 @@ public class ComplexFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(null);
final ColumnValueSelector<?> readSelector =
new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertNull(readSelector.getObject());
}
@ -134,7 +134,7 @@ public class ComplexFieldReaderTest extends InitializedNullHandlingTest
writeToMemory("foo");
final ColumnValueSelector<?> readSelector =
new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals("foo", readSelector.getObject());
}

10
processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java → processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java
View File

@ -22,10 +22,12 @@ package org.apache.druid.frame.field;
public class ConstantFieldPointer implements ReadableFieldPointer
{
private final long position;
private final long length;
public ConstantFieldPointer(long position)
public ConstantFieldPointer(long position, long length)
{
this.position = position;
this.length = length;
}
@Override
@ -33,4 +35,10 @@ public class ConstantFieldPointer implements ReadableFieldPointer
{
return position;
}
@Override
public long length()
{
return length;
}
}

37
processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java
View File

@ -149,10 +149,13 @@ public class DoubleArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_null()
{
writeToMemory(null, MEMORY_POSITION);
long sz = writeToMemory(null, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new DoubleArrayFieldReader().makeColumnValueSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION, sz)
);
Assert.assertTrue(readSelector.isNull());
}
@ -160,10 +163,14 @@ public class DoubleArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_aValue()
{
writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION);
long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new DoubleArrayFieldReader()
.makeColumnValueSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION, sz)
);
assertResults(DOUBLES_LIST_1, readSelector.getObject());
}
@ -172,15 +179,15 @@ public class DoubleArrayFieldReaderTest extends InitializedNullHandlingTest
public void test_makeColumnValueSelector_multipleValues()
{
long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION);
writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz));
long sz2 = writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(
ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz),
ImmutableList.of(sz, sz2)
);
final ColumnValueSelector<?> readSelector = new DoubleArrayFieldReader().makeColumnValueSelector(memory, pointer);
pointer.setPointer(0);
assertResults(DOUBLES_LIST_1, readSelector.getObject());
pointer.setPointer(1);
assertResults(DOUBLES_LIST_2, readSelector.getObject());
}
@ -188,10 +195,11 @@ public class DoubleArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_emptyArray()
{
writeToMemory(new Object[]{}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new DoubleArrayFieldReader()
.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(Collections.emptyList(), readSelector.getObject());
}
@ -199,10 +207,13 @@ public class DoubleArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_arrayWithSingleNullElement()
{
writeToMemory(new Object[]{null}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new DoubleArrayFieldReader().makeColumnValueSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION, sz)
);
assertResults(Collections.singletonList(null), readSelector.getObject());
}

12
processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java
View File

@ -89,7 +89,7 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultDoubleValue());
final ColumnValueSelector<?> readSelector =
DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull());
@ -104,7 +104,7 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5.1d);
final ColumnValueSelector<?> readSelector =
DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(5.1d, readSelector.getObject());
}
@ -115,7 +115,8 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultDoubleValue());
final DimensionSelector readSelector =
DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
DoubleFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -149,7 +150,8 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5.1d);
final DimensionSelector readSelector =
DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
DoubleFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -178,7 +180,7 @@ public class DoubleFieldReaderTest extends InitializedNullHandlingTest
final DimensionSelector readSelector =
DoubleFieldReader.forPrimitive().makeDimensionSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION),
new ConstantFieldPointer(MEMORY_POSITION, -1),
new SubstringDimExtractionFn(1, null)
);

26
processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java
View File

@ -150,10 +150,10 @@ public class FloatArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_null()
{
writeToMemory(null, MEMORY_POSITION);
long sz = writeToMemory(null, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
Assert.assertTrue(readSelector.isNull());
}
@ -161,10 +161,10 @@ public class FloatArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_aValue()
{
writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION);
long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(FLOATS_LIST_1, readSelector.getObject());
}
@ -173,15 +173,15 @@ public class FloatArrayFieldReaderTest extends InitializedNullHandlingTest
public void test_makeColumnValueSelector_multipleValues()
{
long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION);
writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz));
long sz2 = writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(
ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz),
ImmutableList.of(sz, sz2)
);
final ColumnValueSelector<?> readSelector = new FloatArrayFieldReader().makeColumnValueSelector(memory, pointer);
pointer.setPointer(0);
assertResults(FLOATS_LIST_1, readSelector.getObject());
pointer.setPointer(1);
assertResults(FLOATS_LIST_2, readSelector.getObject());
}
@ -189,10 +189,10 @@ public class FloatArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_emptyArray()
{
writeToMemory(new Object[]{}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(Collections.emptyList(), readSelector.getObject());
}
@ -200,10 +200,10 @@ public class FloatArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_arrayWithSingleNullElement()
{
writeToMemory(new Object[]{null}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(Collections.singletonList(null), readSelector.getObject());
}

12
processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java
View File

@ -89,7 +89,7 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultFloatValue());
final ColumnValueSelector<?> readSelector =
FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull());
@ -104,7 +104,7 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5.1f);
final ColumnValueSelector<?> readSelector =
FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(5.1f, readSelector.getObject());
}
@ -115,7 +115,8 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultFloatValue());
final DimensionSelector readSelector =
FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
FloatFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -149,7 +150,8 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5.1f);
final DimensionSelector readSelector =
FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
FloatFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -178,7 +180,7 @@ public class FloatFieldReaderTest extends InitializedNullHandlingTest
final DimensionSelector readSelector =
FloatFieldReader.forPrimitive().makeDimensionSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION),
new ConstantFieldPointer(MEMORY_POSITION, -1),
new SubstringDimExtractionFn(1, null)
);

10
processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java
View File

@ -30,11 +30,13 @@ import java.util.List;
public class IndexArrayFieldPointer implements ReadableFieldPointer
{
private final LongArrayList indices;
private final LongArrayList lengths;
private int pointer = 0;
public IndexArrayFieldPointer(final List<Long> indices)
public IndexArrayFieldPointer(final List<Long> indices, final List<Long> lengths)
{
this.indices = new LongArrayList(indices);
this.lengths = new LongArrayList(lengths);
}
private int numIndices()
@ -53,4 +55,10 @@ public class IndexArrayFieldPointer implements ReadableFieldPointer
{
return indices.getLong(pointer);
}
@Override
public long length()
{
return lengths.getLong(pointer);
}
}

26
processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java
View File

@ -126,10 +126,10 @@ public class LongArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_null()
{
writeToMemory(null, MEMORY_POSITION);
long sz = writeToMemory(null, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
Assert.assertTrue(readSelector.isNull());
}
@ -137,10 +137,10 @@ public class LongArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_aValue()
{
writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION);
long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(LONGS_LIST_1, readSelector.getObject());
}
@ -149,15 +149,15 @@ public class LongArrayFieldReaderTest extends InitializedNullHandlingTest
public void test_makeColumnValueSelector_multipleValues()
{
long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION);
writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz));
long sz2 = writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz);
IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(
ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz),
ImmutableList.of(sz, sz2)
);
final ColumnValueSelector<?> readSelector = new LongArrayFieldReader().makeColumnValueSelector(memory, pointer);
pointer.setPointer(0);
assertResults(LONGS_LIST_1, readSelector.getObject());
pointer.setPointer(1);
assertResults(LONGS_LIST_2, readSelector.getObject());
}
@ -165,10 +165,10 @@ public class LongArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_emptyArray()
{
writeToMemory(new Object[]{}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(Collections.emptyList(), readSelector.getObject());
}
@ -176,10 +176,10 @@ public class LongArrayFieldReaderTest extends InitializedNullHandlingTest
@Test
public void test_makeColumnValueSelector_arrayWithSingleNullElement()
{
writeToMemory(new Object[]{null}, MEMORY_POSITION);
long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION);
final ColumnValueSelector<?> readSelector =
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz));
assertResults(Collections.singletonList(null), readSelector.getObject());
}

12
processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java
View File

@ -89,7 +89,7 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultLongValue());
final ColumnValueSelector<?> readSelector =
LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull());
@ -104,7 +104,7 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5L);
final ColumnValueSelector<?> readSelector =
LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(5L, readSelector.getObject());
}
@ -115,7 +115,8 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(NullHandling.defaultLongValue());
final DimensionSelector readSelector =
LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
LongFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -149,7 +150,8 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(5L);
final DimensionSelector readSelector =
LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
LongFieldReader.forPrimitive()
.makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -178,7 +180,7 @@ public class LongFieldReaderTest extends InitializedNullHandlingTest
final DimensionSelector readSelector =
LongFieldReader.forPrimitive().makeDimensionSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION),
new ConstantFieldPointer(MEMORY_POSITION, -1),
new SubstringDimExtractionFn(1, null)
);

2
processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java
View File

@ -140,7 +140,7 @@ public class StringArrayFieldWriterTest extends InitializedNullHandlingTest
final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY);
final ColumnValueSelector<?> selector =
fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
final Object o = selector.getObject();
//noinspection rawtypes,unchecked

26
processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java
View File

@ -143,9 +143,9 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(Collections.singletonList("foo"));
final ColumnValueSelector<?> readSelector =
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
final ColumnValueSelector<?> readSelectorAsArray =
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals("foo", readSelector.getObject());
Assert.assertArrayEquals(new Object[]{"foo"}, (Object[]) readSelectorAsArray.getObject());
@ -157,9 +157,9 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(ImmutableList.of("foo", "bar"));
final ColumnValueSelector<?> readSelector =
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
final ColumnValueSelector<?> readSelectorAsArray =
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertEquals(ImmutableList.of("foo", "bar"), readSelector.getObject());
Assert.assertArrayEquals(new Object[]{"foo", "bar"}, (Object[]) readSelectorAsArray.getObject());
@ -171,9 +171,9 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(Collections.singletonList(null));
final ColumnValueSelector<?> readSelector =
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
final ColumnValueSelector<?> readSelectorAsArray =
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertNull(readSelector.getObject());
Assert.assertArrayEquals(new Object[]{null}, (Object[]) readSelectorAsArray.getObject());
@ -185,9 +185,9 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(Collections.emptyList());
final ColumnValueSelector<?> readSelector =
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
final ColumnValueSelector<?> readSelectorAsArray =
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
Assert.assertNull(readSelector.getObject());
Assert.assertArrayEquals(ObjectArrays.EMPTY_ARRAY, (Object[]) readSelectorAsArray.getObject());
@ -200,7 +200,11 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
final IllegalStateException e = Assert.assertThrows(
IllegalStateException.class,
() -> new StringArrayFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null)
() -> new StringArrayFieldReader().makeDimensionSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION, -1),
null
)
);
MatcherAssert.assertThat(
@ -215,7 +219,7 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
writeToMemory(ImmutableList.of("foo", "bar"));
final DimensionSelector readSelector =
new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null);
new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null);
// Data retrieval tests.
final IndexedInts row = readSelector.getRow();
@ -247,7 +251,7 @@ public class StringFieldReaderTest extends InitializedNullHandlingTest
final DimensionSelector readSelector =
new StringFieldReader().makeDimensionSelector(
memory,
new ConstantFieldPointer(MEMORY_POSITION),
new ConstantFieldPointer(MEMORY_POSITION, -1),
new SubstringDimExtractionFn(1, null)
);

2
processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java
View File

@ -184,7 +184,7 @@ public class StringFieldWriterTest extends InitializedNullHandlingTest
final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY);
final ColumnValueSelector<?> selector =
fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION));
fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1));
return (Object[]) selector.getObject();
}