diff --git a/core/src/main/java/org/apache/druid/data/input/StringTuple.java b/core/src/main/java/org/apache/druid/data/input/StringTuple.java index 1362ed7f3cf..39d25e3211a 100644 --- a/core/src/main/java/org/apache/druid/data/input/StringTuple.java +++ b/core/src/main/java/org/apache/druid/data/input/StringTuple.java @@ -39,6 +39,15 @@ public class StringTuple implements Comparable return new StringTuple(values); } + /** + * Gets the first String from the given StringTuple if the tuple is non-null + * and non-empty, null otherwise. + */ + public static String firstOrNull(StringTuple tuple) + { + return tuple == null || tuple.size() < 1 ? null : tuple.get(0); + } + @JsonCreator public StringTuple(String[] values) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java index 88f5e85789c..166c7b8e1a0 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java @@ -29,7 +29,12 @@ import java.util.Objects; /** * See {@link BuildingShardSpec} for how this class is used. + *

+ * Calling {@link #convert(int)} on an instance of this class creates a + * {@link SingleDimensionShardSpec} if there is a single dimension or a + * {@link DimensionRangeShardSpec} if there are multiple dimensions. * + * @see SingleDimensionShardSpec * @see DimensionRangeShardSpec */ public class BuildingDimensionRangeShardSpec implements BuildingShardSpec @@ -68,14 +73,14 @@ public class BuildingDimensionRangeShardSpec implements BuildingShardSpec PartitionChunk createChunk(T obj) { diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java index 6dd09920544..1b9c61351c7 100644 --- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java +++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java @@ -21,8 +21,13 @@ package org.apache.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import org.apache.druid.data.input.StringTuple; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; /** @@ -30,17 +35,17 @@ import java.util.Objects; * * @see SingleDimensionShardSpec */ -public class BuildingSingleDimensionShardSpec implements BuildingShardSpec +public class BuildingSingleDimensionShardSpec extends BuildingDimensionRangeShardSpec { public static final String TYPE = "building_single_dim"; - private final int bucketId; private final String dimension; + @Nullable private final String start; + @Nullable private final String end; - private final int partitionId; @JsonCreator public BuildingSingleDimensionShardSpec( @@ -51,57 +56,58 @@ public class BuildingSingleDimensionShardSpec implements BuildingShardSpec getSerializableObject() + { + Map jsonMap = new HashMap<>(); + jsonMap.put("dimension", dimension); + jsonMap.put("start", start); + jsonMap.put("end", end); + jsonMap.put("bucketId", getBucketId()); + jsonMap.put("partitionNum", getPartitionNum()); + + return jsonMap; + } + public String getDimension() { return dimension; } @Nullable - @JsonProperty("start") public String getStart() { return start; } @Nullable - @JsonProperty("end") public String getEnd() { return end; } - @Override - @JsonProperty("partitionNum") - public int getPartitionNum() - { - return partitionId; - } - - @Override - @JsonProperty("bucketId") - public int getBucketId() - { - return bucketId; - } - @Override public SingleDimensionShardSpec convert(int numCorePartitions) { - return new SingleDimensionShardSpec(dimension, start, end, partitionId, numCorePartitions); + return new SingleDimensionShardSpec(dimension, start, end, getPartitionNum(), numCorePartitions); } @Override public PartitionChunk createChunk(T obj) { - return new NumberedPartitionChunk<>(partitionId, 0, obj); + return new NumberedPartitionChunk<>(getPartitionNum(), 0, obj); } @Override @@ -114,8 +120,8 @@ public class BuildingSingleDimensionShardSpec implements BuildingShardSpec(); - } - /** * @param partitions Elements corresponding to evenly-spaced fractional ranks of the distribution */ @@ -71,6 +67,57 @@ public class PartitionBoundaries extends ForwardingList implements delegate = Collections.unmodifiableList(partitionBoundaries); } + /** + * This constructor supports an array of Objects and not just an array of + * StringTuples for backward compatibility. Older versions of this class + * are serialized as a String array. + * + * @param partitions array of StringTuples or array of String + */ + @JsonCreator + private PartitionBoundaries(Object[] partitions) + { + delegate = Arrays.stream(partitions) + .map(this::toStringTuple) + .collect(Collectors.toList()); + } + + @JsonValue + public Object getSerializableObject() + { + boolean isSingleDim = true; + for (StringTuple tuple : delegate) { + if (tuple != null && tuple.size() != 1) { + isSingleDim = false; + break; + } + } + + if (isSingleDim) { + return delegate.stream().map(StringTuple::firstOrNull).collect(Collectors.toList()); + } else { + return delegate; + } + } + + /** + * Converts the given item to a StringTuple. + */ + private StringTuple toStringTuple(Object item) + { + if (item == null || item instanceof StringTuple) { + return (StringTuple) item; + } else if (item instanceof String) { + return StringTuple.create((String) item); + } else if (item instanceof String[]) { + return StringTuple.create((String[]) item); + } else if (item instanceof List) { + return StringTuple.create((String[]) ((List) item).toArray(new String[0])); + } else { + throw new IAE("Item must either be a String or StringTuple"); + } + } + @Override protected List delegate() { diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java index 9817f8d34a2..4c8045aba66 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java @@ -58,13 +58,7 @@ public class BuildingDimensionRangeShardSpecTest public void testConvert_withSingleDimension() { Assert.assertEquals( - new SingleDimensionShardSpec( - "dim", - "start", - "end", - 5, - 10 - ), + new SingleDimensionShardSpec("dim", "start", "end", 5, 10), new BuildingDimensionRangeShardSpec( 1, Collections.singletonList("dim"), diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java index d70a42ff5ba..84ff681cc51 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java @@ -19,16 +19,19 @@ package org.apache.druid.timeline.partition; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.InjectableValues.Std; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; -import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.java.util.common.ISE; import org.junit.Assert; import org.junit.Test; +import java.util.Map; + public class BuildingSingleDimensionShardSpecTest { + private static final ObjectMapper OBJECT_MAPPER = setupObjectMapper(); + @Test public void testConvert() { @@ -48,25 +51,88 @@ public class BuildingSingleDimensionShardSpecTest } @Test - public void testSerde() throws JsonProcessingException + public void testSerde() + { + final BuildingSingleDimensionShardSpec original = + new BuildingSingleDimensionShardSpec(1, "dim", "start", "end", 5); + final String json = serialize(original); + final BuildingSingleDimensionShardSpec fromJson = + (BuildingSingleDimensionShardSpec) deserialize(json, ShardSpec.class); + Assert.assertEquals(original, fromJson); + } + + @Test + public void testGetSerializableObject() + { + BuildingSingleDimensionShardSpec shardSpec = + new BuildingSingleDimensionShardSpec(1, "dim", "abc", "xyz", 5); + + // Verify the fields of the serializable object + Map jsonMap = shardSpec.getSerializableObject(); + Assert.assertEquals(5, jsonMap.size()); + Assert.assertEquals(1, jsonMap.get("bucketId")); + Assert.assertEquals("dim", jsonMap.get("dimension")); + Assert.assertEquals("abc", jsonMap.get("start")); + Assert.assertEquals("xyz", jsonMap.get("end")); + Assert.assertEquals(5, jsonMap.get("partitionNum")); + } + + @Test + public void testDeserializeFromMap() + { + final String json = "{\"type\": \"" + BuildingSingleDimensionShardSpec.TYPE + "\"," + + " \"bucketId\":1," + + " \"dimension\": \"dim\"," + + " \"start\": \"abc\"," + + " \"end\": \"xyz\"," + + " \"partitionNum\": 5}"; + + BuildingSingleDimensionShardSpec shardSpec = + (BuildingSingleDimensionShardSpec) deserialize(json, ShardSpec.class); + Assert.assertEquals( + new BuildingSingleDimensionShardSpec(1, "dim", "abc", "xyz", 5), + shardSpec + ); + } + + @Test + public void testEquals() + { + Assert.assertEquals( + new BuildingSingleDimensionShardSpec(10, "dim", "start", "end", 4), + new BuildingSingleDimensionShardSpec(10, "dim", "start", "end", 4) + ); + } + + private static ObjectMapper setupObjectMapper() { final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper(); mapper.registerSubtypes( new NamedType(BuildingSingleDimensionShardSpec.class, BuildingSingleDimensionShardSpec.TYPE) ); mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper)); - final BuildingSingleDimensionShardSpec original = new BuildingSingleDimensionShardSpec(1, "dim", "start", "end", 5); - final String json = mapper.writeValueAsString(original); - final BuildingSingleDimensionShardSpec fromJson = (BuildingSingleDimensionShardSpec) mapper.readValue( - json, - ShardSpec.class - ); - Assert.assertEquals(original, fromJson); + + return mapper; } - @Test - public void testEquals() + private String serialize(Object object) { - EqualsVerifier.forClass(BuildingSingleDimensionShardSpec.class).usingGetClass().verify(); + try { + return OBJECT_MAPPER.writeValueAsString(object); + } + catch (Exception e) { + throw new ISE("Error while serializing"); + } } + + private T deserialize(String json, Class clazz) + { + try { + return OBJECT_MAPPER.readValue(json, clazz); + } + catch (Exception e) { + throw new ISE(e, "Error while deserializing"); + } + } + } diff --git a/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java index 3285c15964b..16d34dd3ebf 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class DimensionRangeBucketShardSpecTest @@ -65,6 +66,20 @@ public class DimensionRangeBucketShardSpecTest ); } + @Test + public void testConvert_withSingleDimension() + { + Assert.assertEquals( + new BuildingSingleDimensionShardSpec(1, "dim", "start", "end", 5), + new DimensionRangeBucketShardSpec( + 1, + Collections.singletonList("dim"), + StringTuple.create("start"), + StringTuple.create("end") + ).convert(5) + ); + } + @Test public void testCreateChunk() { diff --git a/core/src/test/java/org/apache/druid/timeline/partition/PartitionBoundariesTest.java b/core/src/test/java/org/apache/druid/timeline/partition/PartitionBoundariesTest.java index e87818cd266..96ac623be50 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/PartitionBoundariesTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/PartitionBoundariesTest.java @@ -19,10 +19,10 @@ package org.apache.druid.timeline.partition; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.StringTuple; +import org.apache.druid.java.util.common.ISE; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -34,6 +34,8 @@ import java.util.List; public class PartitionBoundariesTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private PartitionBoundaries target; private StringTuple[] values; private List expected; @@ -79,16 +81,108 @@ public class PartitionBoundariesTest @Test public void handlesRepeatedValue() { - Assert.assertEquals(Arrays.asList(null, null), new PartitionBoundaries(StringTuple.create("a"), StringTuple.create("a"), StringTuple.create("a"))); + Assert.assertEquals( + Arrays.asList(null, null), + new PartitionBoundaries( + StringTuple.create("a"), + StringTuple.create("a"), + StringTuple.create("a") + ) + ); } @Test - public void serializesDeserializes() throws JsonProcessingException + public void serializesDeserializes() { - final ObjectMapper objectMapper = new ObjectMapper(); - String serialized = objectMapper.writeValueAsString(target); - Object deserialized = objectMapper.readValue(serialized, target.getClass()); - Assert.assertEquals(serialized, objectMapper.writeValueAsString(deserialized)); + String serialized = serialize(target); + Object deserialized = deserialize(serialized, target.getClass()); + Assert.assertEquals(serialized, serialize(deserialized)); + } + + @Test + public void testSerdeWithMultiDimensions() + { + PartitionBoundaries original = new PartitionBoundaries( + StringTuple.create("a", "10"), + StringTuple.create("b", "7"), + StringTuple.create("c", "4") + ); + + String json = serialize(original); + PartitionBoundaries deserialized = deserialize(json, PartitionBoundaries.class); + Assert.assertEquals(original, deserialized); + } + + @Test + public void testGetSerializableObject_withMultiDimensions() + { + // Create a PartitionBoundaries for multiple dimensions + PartitionBoundaries multiDimBoundaries = new PartitionBoundaries( + StringTuple.create("a", "10"), + StringTuple.create("b", "7"), + StringTuple.create("c", "4") + ); + + // Verify that the serializable object is a List + Object serializableObject = multiDimBoundaries.getSerializableObject(); + Assert.assertTrue(serializableObject instanceof List); + assertThatItemsAreNullOr(StringTuple.class, (List) serializableObject); + + // Verify the output of getSerializableObject can be serialized/deserialized + String json = serialize(serializableObject); + PartitionBoundaries deserialized = deserialize(json, PartitionBoundaries.class); + Assert.assertEquals(multiDimBoundaries, deserialized); + } + + @Test + public void testGetSerializableObject_withSingleDimension() + { + // Create a PartitionBoundaries for a single dimension + PartitionBoundaries singleDimBoundaries = new PartitionBoundaries( + StringTuple.create("a"), + StringTuple.create("b"), + StringTuple.create("c") + ); + + // Verify that the serializable object is a List + Object serializableObject = singleDimBoundaries.getSerializableObject(); + Assert.assertTrue(serializableObject instanceof List); + assertThatItemsAreNullOr(String.class, (List) serializableObject); + + // Verify the output of getSerializableObject can be serialized/deserialized + String json = serialize(serializableObject); + PartitionBoundaries deserialized = deserialize(json, PartitionBoundaries.class); + Assert.assertEquals(singleDimBoundaries, deserialized); + } + + @Test + public void testDeserializeArrayOfString() + { + String json = "[null, \"a\", null]"; + PartitionBoundaries deserialized = deserialize(json, PartitionBoundaries.class); + Assert.assertEquals( + new PartitionBoundaries( + null, + StringTuple.create("a"), + StringTuple.create("b") + ), + deserialized + ); + } + + @Test + public void testDeserializeArrayOfTuples() + { + String json = "[null, [\"a\",\"10\"], null]"; + PartitionBoundaries deserialized = deserialize(json, PartitionBoundaries.class); + Assert.assertEquals( + new PartitionBoundaries( + null, + StringTuple.create("a", "10"), + StringTuple.create("a", "20") + ), + deserialized + ); } @Test @@ -102,4 +196,41 @@ public class PartitionBoundariesTest { EqualsVerifier.forClass(PartitionBoundaries.class).withNonnullFields("delegate").usingGetClass().verify(); } + + /** + * Asserts that all the items in the given list are either null or of the + * specified class. + */ + private void assertThatItemsAreNullOr(Class clazz, List list) + { + if (list == null || list.isEmpty()) { + return; + } + + for (Object item : list) { + if (item != null) { + Assert.assertSame(clazz, item.getClass()); + } + } + } + + private String serialize(Object object) + { + try { + return OBJECT_MAPPER.writeValueAsString(object); + } + catch (Exception e) { + throw new ISE("Error while serializing"); + } + } + + private T deserialize(String json, Class clazz) + { + try { + return OBJECT_MAPPER.readValue(json, clazz); + } + catch (Exception e) { + throw new ISE(e, "Error while deserializing"); + } + } } diff --git a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java index 1a05f12e7cf..219c0ea3e65 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java @@ -19,6 +19,7 @@ package org.apache.druid.timeline.partition; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -163,6 +164,25 @@ public class SingleDimensionShardSpecTest testSerde(new SingleDimensionShardSpec("dim", "abc", "xyz", 10, null)); } + @Test + public void testDeserialize() throws JsonProcessingException + { + final String json = "{\"type\": \"single\"," + + " \"dimension\": \"dim\"," + + " \"start\": \"abc\"," + + "\"end\": \"xyz\"," + + "\"partitionNum\": 5," + + "\"numCorePartitions\": 10}"; + ShardSpec shardSpec = OBJECT_MAPPER.readValue(json, ShardSpec.class); + Assert.assertTrue(shardSpec instanceof SingleDimensionShardSpec); + + SingleDimensionShardSpec singleDimShardSpec = (SingleDimensionShardSpec) shardSpec; + Assert.assertEquals( + new SingleDimensionShardSpec("dim", "abc", "xyz", 5, 10), + singleDimShardSpec + ); + } + private void testSerde(SingleDimensionShardSpec shardSpec) throws IOException { String json = OBJECT_MAPPER.writeValueAsString(shardSpec);