From 2f7f5119cf939d1d4a03e804c901ad9f4ca765d7 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 22 Jan 2016 10:18:10 -0600 Subject: [PATCH 1/2] older segments might not have field bitmapSerdeFactory for dimension columns and we must use appropriate default --- .../DictionaryEncodedColumnPartSerde.java | 7 +- .../DictionaryEncodedColumnPartSerdeTest.java | 73 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 processing/src/test/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerdeTest.java diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 43cb2fc9753..650443da245 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -31,6 +31,7 @@ import io.druid.segment.CompressedVSizeIndexedV3Supplier; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferSerializer; import io.druid.segment.data.ByteBufferWriter; @@ -93,7 +94,11 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde @NotNull @JsonProperty("byteOrder") ByteOrder byteOrder ) { - return new DictionaryEncodedColumnPartSerde(byteOrder, bitmapSerdeFactory, null); + return new DictionaryEncodedColumnPartSerde( + byteOrder, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), + null + ); } private final ByteOrder byteOrder; diff --git a/processing/src/test/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerdeTest.java new file mode 100644 index 00000000000..2a0f647fdcd --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerdeTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.ConciseBitmapSerdeFactory; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteOrder; + +/** + */ +public class DictionaryEncodedColumnPartSerdeTest +{ + @Test + public void testSerde() throws Exception + { + //bitmapSerdeFactory not specified + String json = "{\n" + + " \"type\": \"stringDictionary\",\n" + + " \"byteOrder\": \"BIG_ENDIAN\"\n" + + "}\n"; + + ObjectMapper mapper = TestHelper.getObjectMapper(); + + DictionaryEncodedColumnPartSerde serde = (DictionaryEncodedColumnPartSerde) mapper.readValue( + mapper.writeValueAsString( + mapper.readValue(json, ColumnPartSerde.class) + ), + ColumnPartSerde.class + ); + + Assert.assertEquals(ByteOrder.BIG_ENDIAN, serde.getByteOrder()); + Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof ConciseBitmapSerdeFactory); + + //bitmapSerdeFactory specified + json = "{\n" + + "\"type\": \"stringDictionary\",\n" + + "\"byteOrder\": \"LITTLE_ENDIAN\",\n" + + "\"bitmapSerdeFactory\": { \"type\": \"roaring\" }\n" + + "}"; + + serde = (DictionaryEncodedColumnPartSerde) mapper.readValue( + mapper.writeValueAsString( + mapper.readValue(json, ColumnPartSerde.class) + ), + ColumnPartSerde.class + ); + + Assert.assertEquals(ByteOrder.LITTLE_ENDIAN, serde.getByteOrder()); + Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof RoaringBitmapSerdeFactory); + } +} From dccf6366762a83a752f590a0b269aaf0c8993f58 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 22 Jan 2016 11:34:22 -0600 Subject: [PATCH 2/2] adding null aggregators and false hasMultipleValues to expected results of segment metadata query --- .../resources/queries/twitterstream_queries.json | 12 +++++++++--- .../queries/wikipedia_editstream_queries.json | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/integration-tests/src/test/resources/queries/twitterstream_queries.json b/integration-tests/src/test/resources/queries/twitterstream_queries.json index f64ea0df0e1..8e9510fbd7a 100644 --- a/integration-tests/src/test/resources/queries/twitterstream_queries.json +++ b/integration-tests/src/test/resources/queries/twitterstream_queries.json @@ -594,13 +594,15 @@ "columns": { "has_links": { "type": "STRING", + "hasMultipleValues": false, "size": 7773438, "cardinality": 2, "errorMessage": null } }, "size": 747056474, - "numRows": 3702583 + "numRows": 3702583, + "aggregators": null }, { "id": "twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9", @@ -608,13 +610,15 @@ "columns": { "has_links": { "type": "STRING", + "hasMultipleValues": false, "size": 7901000, "cardinality": 2, "errorMessage": null } }, "size": 755796690, - "numRows": 3743002 + "numRows": 3743002, + "aggregators": null }, { "id": "twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9", @@ -622,13 +626,15 @@ "columns": { "has_links": { "type": "STRING", + "hasMultipleValues": false, "size": 7405654, "cardinality": 2, "errorMessage": null } }, "size": 706893542, - "numRows":3502959 + "numRows":3502959, + "aggregators": null } ] }, diff --git a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json index 00056f7899a..0b4572c1133 100644 --- a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json +++ b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json @@ -1045,19 +1045,22 @@ "columns": { "country_name": { "type": "STRING", + "hasMultipleValues": false, "size": 41922148, "cardinality": 208, "errorMessage": null }, "language": { "type": "STRING", + "hasMultipleValues": false, "size": 8924222, "cardinality": 36, "errorMessage": null } }, "size": 902457341, - "numRows": 4462111 + "numRows": 4462111, + "aggregators": null } ] }