diff --git a/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java b/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java index f4e62ff77c2..c67094a285f 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java +++ b/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.base.Function; import com.google.common.collect.Lists; import com.metamx.common.IAE; - import org.joda.time.Interval; import java.util.Arrays; @@ -39,6 +38,8 @@ public class LegacySegmentSpec extends MultipleIntervalSegmentSpec final List intervalStringList; if (intervals instanceof String) { intervalStringList = Arrays.asList((((String) intervals).split(","))); + } else if (intervals instanceof Interval) { + intervalStringList = Arrays.asList(intervals.toString()); } else if (intervals instanceof Map) { intervalStringList = (List) ((Map) intervals).get("intervals"); } else if (intervals instanceof List) { diff --git a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java index 695b116c5b0..62eba78ff61 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -292,7 +292,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter @Override public int getValueCardinality() { - return dimValLookup.size(); + return maxId; } @Override diff --git a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java b/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java index 38bfced4d62..c2d46e54d45 100644 --- a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java +++ b/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -44,6 +44,7 @@ import com.metamx.druid.index.v1.processing.Cursor; import com.metamx.druid.index.v1.processing.DimensionSelector; import com.metamx.druid.input.MapBasedRow; import com.metamx.druid.input.Row; +import com.metamx.druid.kv.IndexedInts; import com.metamx.druid.query.dimension.DimensionSpec; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -173,17 +174,28 @@ public class GroupByQueryEngine { if (dims.size() > 0) { List retVal = null; - for (Integer dimValue : dims.get(0).getRow()) { + List unaggregatedBuffers = null; + + final DimensionSelector dimSelector = dims.get(0); + final IndexedInts row = dimSelector.getRow(); + if (row.size() == 0) { ByteBuffer newKey = key.duplicate(); - newKey.putInt(dimValue); - final List unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); - if (unaggregatedBuffers != null) { - if (retVal == null) { - retVal = Lists.newArrayList(); - } - retVal.addAll(unaggregatedBuffers); + newKey.putInt(dimSelector.getValueCardinality()); + unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); + } + else { + for (Integer dimValue : row) { + ByteBuffer newKey = key.duplicate(); + newKey.putInt(dimValue); + unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); } } + if (unaggregatedBuffers != null) { + if (retVal == null) { + retVal = Lists.newArrayList(); + } + retVal.addAll(unaggregatedBuffers); + } return retVal; } else { @@ -379,7 +391,11 @@ public class GroupByQueryEngine ByteBuffer keyBuffer = input.getKey().duplicate(); for (int i = 0; i < dimensions.size(); ++i) { - theEvent.put(dimNames.get(i), dimensions.get(i).lookupName(keyBuffer.getInt())); + final DimensionSelector dimSelector = dimensions.get(i); + final int dimVal = keyBuffer.getInt(); + if (dimSelector.getValueCardinality() != dimVal) { + theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal)); + } } int position = input.getValue(); diff --git a/server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java b/server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java new file mode 100644 index 00000000000..0072c27d73e --- /dev/null +++ b/server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java @@ -0,0 +1,116 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package com.metamx.druid.index.v1; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import com.metamx.druid.QueryGranularity; +import com.metamx.druid.aggregation.AggregatorFactory; +import com.metamx.druid.aggregation.CountAggregatorFactory; +import com.metamx.druid.aggregation.LongSumAggregatorFactory; +import com.metamx.druid.collect.StupidPool; +import com.metamx.druid.input.MapBasedInputRow; +import com.metamx.druid.input.MapBasedRow; +import com.metamx.druid.input.Row; +import com.metamx.druid.query.group.GroupByQuery; +import com.metamx.druid.query.group.GroupByQueryEngine; +import com.metamx.druid.query.group.GroupByQueryEngineConfig; +import junit.framework.Assert; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; + +/** + */ +public class IncrementalIndexStorageAdapterTest +{ + @Test + public void testSanity() throws Exception + { + IncrementalIndex index = new IncrementalIndex( + 0, QueryGranularity.MINUTE, new AggregatorFactory[]{new CountAggregatorFactory("cnt")} + ); + + index.add( + new MapBasedInputRow( + new DateTime().minus(1).getMillis(), + Lists.newArrayList("billy"), + ImmutableMap.of("billy", "hi") + ) + ); + index.add( + new MapBasedInputRow( + new DateTime().minus(1).getMillis(), + Lists.newArrayList("sally"), + ImmutableMap.of("sally", "bo") + ) + ); + + GroupByQueryEngine engine = new GroupByQueryEngine( + new GroupByQueryEngineConfig() + { + @Override + public int getMaxIntermediateRows() + { + return 5; + } + }, + new StupidPool( + new Supplier() + { + @Override + public ByteBuffer get() + { + return ByteBuffer.allocate(50000); + } + } + ) + ); + + final Sequence rows = engine.process( + GroupByQuery.builder() + .setDataSource("test") + .setGranularity(QueryGranularity.ALL) + .setInterval(new Interval(0, new DateTime().getMillis())) + .addDimension("billy") + .addDimension("sally") + .addAggregator(new LongSumAggregatorFactory("cnt", "cnt")) + .build(), + new IncrementalIndexStorageAdapter(index) + ); + + final ArrayList results = Sequences.toList(rows, Lists.newArrayList()); + + Assert.assertEquals(2, results.size()); + + MapBasedRow row = (MapBasedRow) results.get(0); + Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1l), row.getEvent()); + + row = (MapBasedRow) results.get(1); + Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1l), row.getEvent()); + } + +}