mirror of https://github.com/apache/druid.git
add backwards compat mode for frontCoded stringEncodingStrategy (#13988)
This commit is contained in:
Clint Wylie
GitHub
parent
76fe26d4ba
commit
2219e68fa3
|
|
@ -139,7 +139,7 @@ public class FrontCodedIndexedBenchmark
|
|||
new OnHeapMemorySegmentWriteOutMedium(),
|
||||
ByteOrder.nativeOrder(),
|
||||
"front-coded-4".equals(indexType) ? 4 : 16,
|
||||
false
|
||||
FrontCodedIndexed.V0
|
||||
);
|
||||
frontCodedIndexedWriter.open();
|
||||
|
||||
|
|
@ -147,7 +147,7 @@ public class FrontCodedIndexedBenchmark
|
|||
new OnHeapMemorySegmentWriteOutMedium(),
|
||||
ByteOrder.nativeOrder(),
|
||||
"front-coded-incremental-buckets-4".equals(indexType) ? 4 : 16,
|
||||
true
|
||||
FrontCodedIndexed.V1
|
||||
);
|
||||
frontCodedIndexedWriterIncrementalBuckets.open();
|
||||
|
||||
|
|
|
|||
|
|
@ -477,7 +477,7 @@ The `indexSpec` object can include the following properties:
|
|||
|-----|-----------|-------|
|
||||
|bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`|
|
||||
|dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
|
||||
|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /><br />See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
|
||||
|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /> `formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to `1`<br /><br />See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
|
||||
|metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`|
|
||||
|longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`|
|
||||
|jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
|
||||
|
|
@ -488,7 +488,7 @@ Front coding is an experimental feature starting in version 25.0. Front coding i
|
|||
|
||||
You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters).
|
||||
|
||||
> Front coding is new to Druid 25.0 so the current recommendation is to enable it in a staging environment and fully test your use case before using in production. Segments created with front coding enabled are not compatible with Druid versions older than 25.0.
|
||||
> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are not backwards compatible with Druid 25.0, and those created with Druid 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the `formatVersion` can be specified as `0` keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later changed to `1` (or removed) once determined that rollback is not necessary.
|
||||
|
||||
Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each
|
||||
[ingestion method](./index.md#ingestion-methods) for details.
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ public class StringEncodingStrategies
|
|||
writeoutMedium,
|
||||
IndexIO.BYTE_ORDER,
|
||||
strategy.getBucketSize(),
|
||||
true
|
||||
strategy.getFormatVersion()
|
||||
);
|
||||
} else {
|
||||
throw new ISE("Unknown encoding strategy: %s", encodingStrategy.getType());
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
|||
import com.fasterxml.jackson.annotation.JsonSubTypes;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
import org.apache.druid.java.util.common.ISE;
|
||||
import org.apache.druid.segment.data.FrontCodedIndexed;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Objects;
|
||||
|
|
@ -84,20 +85,31 @@ public interface StringEncodingStrategy
|
|||
|
||||
class FrontCoded implements StringEncodingStrategy
|
||||
{
|
||||
public static final int DEFAULT_BUCKET_SIZE = 4;
|
||||
|
||||
@JsonProperty
|
||||
private final int bucketSize;
|
||||
|
||||
@JsonProperty
|
||||
private final byte formatVersion;
|
||||
|
||||
@JsonCreator
|
||||
public FrontCoded(
|
||||
@JsonProperty("bucketSize") @Nullable Integer bucketSize
|
||||
@JsonProperty("bucketSize") @Nullable Integer bucketSize,
|
||||
@JsonProperty("formatVersion") @Nullable Byte version
|
||||
)
|
||||
{
|
||||
this.bucketSize = bucketSize == null ? DEFAULT_BUCKET_SIZE : bucketSize;
|
||||
this.bucketSize = bucketSize == null ? FrontCodedIndexed.DEFAULT_BUCKET_SIZE : bucketSize;
|
||||
if (Integer.bitCount(this.bucketSize) != 1) {
|
||||
throw new ISE("bucketSize must be a power of two but was[%,d]", bucketSize);
|
||||
}
|
||||
this.formatVersion = version == null
|
||||
? FrontCodedIndexed.DEFAULT_VERSION
|
||||
: FrontCodedIndexed.validateVersion(version);
|
||||
}
|
||||
|
||||
public FrontCoded(@Nullable Integer bucketSize)
|
||||
{
|
||||
this(bucketSize, null);
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -106,6 +118,12 @@ public interface StringEncodingStrategy
|
|||
return bucketSize;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public byte getFormatVersion()
|
||||
{
|
||||
return formatVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType()
|
||||
{
|
||||
|
|
@ -128,13 +146,13 @@ public interface StringEncodingStrategy
|
|||
return false;
|
||||
}
|
||||
FrontCoded that = (FrontCoded) o;
|
||||
return bucketSize == that.bucketSize;
|
||||
return bucketSize == that.bucketSize && formatVersion == that.formatVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(bucketSize);
|
||||
return Objects.hash(bucketSize, formatVersion);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -142,6 +160,7 @@ public interface StringEncodingStrategy
|
|||
{
|
||||
return "FrontCoded{" +
|
||||
"bucketSize=" + bucketSize +
|
||||
", formatVersion=" + formatVersion +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
|
|||
import com.google.common.base.Supplier;
|
||||
import org.apache.druid.annotations.SuppressFBWarnings;
|
||||
import org.apache.druid.common.config.NullHandling;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.java.util.common.ISE;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
|
||||
|
|
@ -76,11 +77,28 @@ import java.util.NoSuchElementException;
|
|||
*/
|
||||
public final class FrontCodedIndexed implements Indexed<ByteBuffer>
|
||||
{
|
||||
public static final byte V0 = 0;
|
||||
public static final byte V1 = 1;
|
||||
public static final byte DEFAULT_VERSION = V1;
|
||||
public static final int DEFAULT_BUCKET_SIZE = 4;
|
||||
|
||||
public static byte validateVersion(byte version)
|
||||
{
|
||||
if (version != FrontCodedIndexed.V0 && version != FrontCodedIndexed.V1) {
|
||||
throw new IAE(
|
||||
"Unknown format version for FrontCodedIndexed [%s], must be [%s] or [%s]",
|
||||
version,
|
||||
FrontCodedIndexed.V0,
|
||||
FrontCodedIndexed.V1
|
||||
);
|
||||
}
|
||||
return version;
|
||||
}
|
||||
public static Supplier<FrontCodedIndexed> read(ByteBuffer buffer, ByteOrder ordering)
|
||||
{
|
||||
final ByteBuffer orderedBuffer = buffer.asReadOnlyBuffer().order(ordering);
|
||||
final byte version = orderedBuffer.get();
|
||||
Preconditions.checkArgument(version == 0 || version == 1, "only V0 and V1 exist, encountered " + version);
|
||||
Preconditions.checkArgument(version == V0 || version == V1, "only V0 and V1 exist, encountered " + version);
|
||||
final int bucketSize = Byte.toUnsignedInt(orderedBuffer.get());
|
||||
final boolean hasNull = NullHandling.IS_NULL_BYTE == orderedBuffer.get();
|
||||
final int numValues = VByte.readInt(orderedBuffer);
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
private final byte[][] bucketBuffer;
|
||||
private final ByteBuffer getOffsetBuffer;
|
||||
private final int div;
|
||||
private final boolean useIncrementalBuckets;
|
||||
private final byte version;
|
||||
|
||||
@Nullable
|
||||
private byte[] prevObject = null;
|
||||
|
|
@ -78,7 +78,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
SegmentWriteOutMedium segmentWriteOutMedium,
|
||||
ByteOrder byteOrder,
|
||||
int bucketSize,
|
||||
boolean useIncrementalBuckets
|
||||
byte version
|
||||
)
|
||||
{
|
||||
if (Integer.bitCount(bucketSize) != 1 || bucketSize < 1 || bucketSize > 128) {
|
||||
|
|
@ -91,7 +91,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
this.bucketBuffer = new byte[bucketSize][];
|
||||
this.getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder);
|
||||
this.div = Integer.numberOfTrailingZeros(bucketSize);
|
||||
this.useIncrementalBuckets = useIncrementalBuckets;
|
||||
this.version = FrontCodedIndexed.validateVersion(version);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -124,9 +124,9 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
int written;
|
||||
// write the bucket, growing scratch buffer as necessary
|
||||
do {
|
||||
written = useIncrementalBuckets
|
||||
? writeIncrementalBucket(scratch, bucketBuffer, bucketSize)
|
||||
: writeBucket(scratch, bucketBuffer, bucketSize);
|
||||
written = version == FrontCodedIndexed.V1
|
||||
? writeBucketV1(scratch, bucketBuffer, bucketSize)
|
||||
: writeBucketV0(scratch, bucketBuffer, bucketSize);
|
||||
if (written < 0) {
|
||||
growScratch();
|
||||
}
|
||||
|
|
@ -170,14 +170,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
flush();
|
||||
}
|
||||
resetScratch();
|
||||
|
||||
if (useIncrementalBuckets) {
|
||||
// version 1 is incremental buckets
|
||||
scratch.put((byte) 1);
|
||||
} else {
|
||||
// version 0 all values are prefixed on first bucket value
|
||||
scratch.put((byte) 0);
|
||||
}
|
||||
scratch.put(version);
|
||||
scratch.put((byte) bucketSize);
|
||||
scratch.put(hasNulls ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
|
||||
VByte.writeInt(scratch, numWritten);
|
||||
|
|
@ -222,7 +215,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
final ByteBuffer bucketBuffer = ByteBuffer.allocate(bucketBytesSize).order(byteOrder);
|
||||
valuesOut.readFully(startOffset, bucketBuffer);
|
||||
bucketBuffer.clear();
|
||||
final ByteBuffer valueBuffer = useIncrementalBuckets
|
||||
final ByteBuffer valueBuffer = version == FrontCodedIndexed.V1
|
||||
? getFromBucketV1(bucketBuffer, relativeIndex, bucketSize)
|
||||
: FrontCodedIndexed.getFromBucketV0(bucketBuffer, relativeIndex);
|
||||
final byte[] valueBytes = new byte[valueBuffer.limit() - valueBuffer.position()];
|
||||
|
|
@ -248,9 +241,9 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
int written;
|
||||
do {
|
||||
int flushSize = remainder == 0 ? bucketSize : remainder;
|
||||
written = useIncrementalBuckets
|
||||
? writeIncrementalBucket(scratch, bucketBuffer, flushSize)
|
||||
: writeBucket(scratch, bucketBuffer, flushSize);
|
||||
written = version == FrontCodedIndexed.V1
|
||||
? writeBucketV1(scratch, bucketBuffer, flushSize)
|
||||
: writeBucketV0(scratch, bucketBuffer, flushSize);
|
||||
if (written < 0) {
|
||||
growScratch();
|
||||
}
|
||||
|
|
@ -283,7 +276,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
*
|
||||
* Uses {@link VByte} encoded integers to indicate prefix length and value length.
|
||||
*/
|
||||
public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues)
|
||||
public static int writeBucketV0(ByteBuffer buffer, byte[][] values, int numValues)
|
||||
{
|
||||
int written = 0;
|
||||
byte[] first = null;
|
||||
|
|
@ -333,7 +326,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
|
|||
*
|
||||
* Uses {@link VByte} encoded integers to indicate prefix length and value length.
|
||||
*/
|
||||
public static int writeIncrementalBucket(ByteBuffer buffer, byte[][] values, int numValues)
|
||||
public static int writeBucketV1(ByteBuffer buffer, byte[][] values, int numValues)
|
||||
{
|
||||
int written = 0;
|
||||
byte[] prev = null;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.druid.segment.column;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import nl.jqno.equalsverifier.EqualsVerifier;
|
||||
import org.apache.druid.jackson.DefaultObjectMapper;
|
||||
import org.apache.druid.segment.data.FrontCodedIndexed;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class StringEncodingStrategyTest
|
||||
{
|
||||
private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
|
||||
|
||||
|
||||
@Test
|
||||
public void testUtf8Serde() throws JsonProcessingException
|
||||
{
|
||||
StringEncodingStrategy utf8 = new StringEncodingStrategy.Utf8();
|
||||
String there = JSON_MAPPER.writeValueAsString(utf8);
|
||||
StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
|
||||
Assert.assertEquals(utf8, andBackAgain);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFrontCodedDefaultSerde() throws JsonProcessingException
|
||||
{
|
||||
StringEncodingStrategy frontCoded = new StringEncodingStrategy.FrontCoded(null, null);
|
||||
String there = JSON_MAPPER.writeValueAsString(frontCoded);
|
||||
StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
|
||||
Assert.assertEquals(frontCoded, andBackAgain);
|
||||
Assert.assertEquals(FrontCodedIndexed.DEFAULT_BUCKET_SIZE, ((StringEncodingStrategy.FrontCoded) andBackAgain).getBucketSize());
|
||||
Assert.assertEquals(FrontCodedIndexed.DEFAULT_VERSION, ((StringEncodingStrategy.FrontCoded) andBackAgain).getFormatVersion());
|
||||
|
||||
// this next assert seems silly, but its a sanity check to make us think hard before changing the default version,
|
||||
// to make us think of the backwards compatibility implications, as new versions of segment format stuff cannot be
|
||||
// downgraded to older versions of Druid and still read
|
||||
Assert.assertEquals(FrontCodedIndexed.V1, FrontCodedIndexed.DEFAULT_VERSION);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFrontCodedFormatSerde() throws JsonProcessingException
|
||||
{
|
||||
StringEncodingStrategy frontCodedV0 = new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0);
|
||||
String there = JSON_MAPPER.writeValueAsString(frontCodedV0);
|
||||
StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
|
||||
Assert.assertEquals(frontCodedV0, andBackAgain);
|
||||
|
||||
StringEncodingStrategy frontCodedV1 = new StringEncodingStrategy.FrontCoded(8, FrontCodedIndexed.V1);
|
||||
there = JSON_MAPPER.writeValueAsString(frontCodedV1);
|
||||
andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
|
||||
Assert.assertEquals(frontCodedV1, andBackAgain);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEqualsAndHashcode()
|
||||
{
|
||||
EqualsVerifier.forClass(StringEncodingStrategy.Utf8.class).usingGetClass().verify();
|
||||
EqualsVerifier.forClass(StringEncodingStrategy.FrontCoded.class).usingGetClass().verify();
|
||||
}
|
||||
}
|
||||
|
|
@ -47,20 +47,20 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
public static Collection<Object[]> constructorFeeder()
|
||||
{
|
||||
return ImmutableList.of(
|
||||
new Object[]{ByteOrder.LITTLE_ENDIAN, true},
|
||||
new Object[]{ByteOrder.LITTLE_ENDIAN, false},
|
||||
new Object[]{ByteOrder.BIG_ENDIAN, true},
|
||||
new Object[]{ByteOrder.BIG_ENDIAN, false}
|
||||
new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V1},
|
||||
new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V0},
|
||||
new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V1},
|
||||
new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V0}
|
||||
);
|
||||
}
|
||||
|
||||
private final ByteOrder order;
|
||||
private final boolean useIncrementalBuckets;
|
||||
private final byte version;
|
||||
|
||||
public FrontCodedIndexedTest(ByteOrder byteOrder, boolean useIncrementalBuckets)
|
||||
public FrontCodedIndexedTest(ByteOrder byteOrder, byte version)
|
||||
{
|
||||
this.order = byteOrder;
|
||||
this.useIncrementalBuckets = useIncrementalBuckets;
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -68,7 +68,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
{
|
||||
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
|
||||
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
|
||||
persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 4, version);
|
||||
|
||||
buffer.position(0);
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
|
|
@ -99,7 +99,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
{
|
||||
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
|
||||
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
|
||||
persistToBuffer(buffer, theList, 16, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 16, version);
|
||||
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
|
|
@ -137,7 +137,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
|
||||
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
|
||||
}
|
||||
persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, values, bucketSize, version);
|
||||
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
|
|
@ -173,7 +173,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
|
||||
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
|
||||
}
|
||||
persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, values, bucketSize, version);
|
||||
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
|
|
@ -207,7 +207,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
|
||||
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
|
||||
|
||||
persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 4, version);
|
||||
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
|
|
@ -231,7 +231,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
TreeSet<String> values = new TreeSet<>(GenericIndexed.STRING_STRATEGY);
|
||||
values.add(null);
|
||||
values.addAll(theList);
|
||||
persistToBuffer(buffer, values, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, values, 4, version);
|
||||
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
|
|
@ -254,7 +254,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
|
||||
// "\uD83D\uDCA9" and "（請參見已被刪除版本）" are a regression test for https://github.com/apache/druid/pull/13364
|
||||
List<String> theList = ImmutableList.of("Győ-Moson-Sopron", "Győr", "\uD83D\uDCA9", "（請參見已被刪除版本）");
|
||||
persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 4, version);
|
||||
|
||||
buffer.position(0);
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
|
|
@ -282,7 +282,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
{
|
||||
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
|
||||
List<String> theList = Collections.singletonList(null);
|
||||
persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 4, version);
|
||||
|
||||
buffer.position(0);
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
|
|
@ -308,7 +308,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
{
|
||||
ByteBuffer buffer = ByteBuffer.allocate(1 << 6).order(order);
|
||||
List<String> theList = Collections.emptyList();
|
||||
persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, theList, 4, version);
|
||||
|
||||
buffer.position(0);
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
|
|
@ -351,7 +351,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
|
||||
}
|
||||
for (int bucketSize : bucketSizes) {
|
||||
persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
|
||||
persistToBuffer(buffer, values, bucketSize, version);
|
||||
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
|
||||
buffer,
|
||||
buffer.order()
|
||||
|
|
@ -389,7 +389,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
medium,
|
||||
ByteOrder.nativeOrder(),
|
||||
0,
|
||||
useIncrementalBuckets
|
||||
version
|
||||
)
|
||||
);
|
||||
|
||||
|
|
@ -399,7 +399,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
medium,
|
||||
ByteOrder.nativeOrder(),
|
||||
15,
|
||||
useIncrementalBuckets
|
||||
version
|
||||
)
|
||||
);
|
||||
|
||||
|
|
@ -409,12 +409,17 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
medium,
|
||||
ByteOrder.nativeOrder(),
|
||||
256,
|
||||
useIncrementalBuckets
|
||||
version
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private static long persistToBuffer(ByteBuffer buffer, Iterable<String> sortedIterable, int bucketSize, boolean useIncrementalBuckets) throws IOException
|
||||
private static long persistToBuffer(
|
||||
ByteBuffer buffer,
|
||||
Iterable<String> sortedIterable,
|
||||
int bucketSize,
|
||||
byte version
|
||||
) throws IOException
|
||||
{
|
||||
Iterator<String> sortedStrings = sortedIterable.iterator();
|
||||
buffer.position(0);
|
||||
|
|
@ -424,7 +429,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
|
|||
medium,
|
||||
buffer.order(),
|
||||
bucketSize,
|
||||
useIncrementalBuckets
|
||||
version
|
||||
);
|
||||
writer.open();
|
||||
int index = 0;
|
||||
|
|
|
|||
Loading…
Reference in New Issue