diff --git a/common/pom.xml b/common/pom.xml index 1716a004688..af29a79f989 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/examples/bin/examples/wikipedia/query.body b/examples/bin/examples/wikipedia/query.body index 4407ccceaa8..fd37209b7df 100644 --- a/examples/bin/examples/wikipedia/query.body +++ b/examples/bin/examples/wikipedia/query.body @@ -12,8 +12,8 @@ }, { "type":"longSum", - "fieldName":"edits", - "name":"edits" + "fieldName":"edit_count", + "name":"count" } ], "filter":{ diff --git a/examples/bin/examples/wikipedia/wikipedia_realtime.spec b/examples/bin/examples/wikipedia/wikipedia_realtime.spec index ae0949acd6d..3ed884ac9ea 100644 --- a/examples/bin/examples/wikipedia/wikipedia_realtime.spec +++ b/examples/bin/examples/wikipedia/wikipedia_realtime.spec @@ -2,7 +2,7 @@ "schema": { "dataSource": "wikipedia", "aggregators": [ - {"type": "count", "name": "edits"}, + {"type": "count", "name": "count"}, {"type": "longSum", "fieldName": "added", "name": "added"}, {"type": "longSum", "fieldName": "deleted", "name": "deleted"}, {"type": "longSum", "fieldName": "delta", "name": "delta"} diff --git a/examples/bin/run_example_client.sh b/examples/bin/run_example_client.sh index a821934d2fd..316034bec48 100755 --- a/examples/bin/run_example_client.sh +++ b/examples/bin/run_example_client.sh @@ -45,7 +45,7 @@ for delay in 5 30 30 30 30 30 30 30 30 30 30 echo "sleep for $delay seconds..." echo " " sleep $delay - curl -X POST 'http://localhost:8080/druid/v2/?w' -H 'content-type: application/json' -d "`cat ${QUERY_FILE}`" + curl -X POST 'http://localhost:8083/druid/v2/?w' -H 'content-type: application/json' -d "`cat ${QUERY_FILE}`" echo " " echo " " done diff --git a/examples/bin/run_example_server.sh b/examples/bin/run_example_server.sh index 6d513f6b453..dc0483b932a 100755 --- a/examples/bin/run_example_server.sh +++ b/examples/bin/run_example_server.sh @@ -58,7 +58,7 @@ DRUID_CP=${EXAMPLE_LOC} DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/../target/druid-examples-*-selfcontained.jar` DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/../config/realtime #For the kit -DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/lib/druid-examples-*-selfcontained.jar` +DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/lib/* DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/config/realtime echo "Running command:" diff --git a/examples/config/realtime/runtime.properties b/examples/config/realtime/runtime.properties index b7366c20595..95e73cc5d12 100644 --- a/examples/config/realtime/runtime.properties +++ b/examples/config/realtime/runtime.properties @@ -6,11 +6,16 @@ druid.service=realtime com.metamx.emitter.logging=true com.metamx.emitter.logging.level=info +# zk +druid.zk.service.host=localhost +druid.zk.paths.base=/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath + # processing druid.processing.buffer.sizeBytes=10000000 # schema -druid.realtime.specFile=realtime.spec +druid.realtime.specFile=config/realtime/realtime.spec # aws (dummy user) com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ diff --git a/examples/pom.xml b/examples/pom.xml index 54155a54ce3..2a8310e92ba 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -1,4 +1,23 @@ + + 4.0.0 com.metamx.druid @@ -9,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 1571eacff50..569a61db92b 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 400380b7899..30c847821de 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 545e33c684d..f416946ff99 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT druid druid @@ -38,7 +38,7 @@ UTF-8 - 0.25.0-SNAPSHOT + 0.25.0 2.1.0-incubating @@ -65,7 +65,7 @@ com.metamx emitter - 0.2.5-SNAPSHOT + 0.2.5 com.metamx @@ -492,7 +492,7 @@ maven-shade-plugin - 1.7.1 + 2.1 maven-site-plugin diff --git a/processing/pom.xml b/processing/pom.xml index dbca7dfc46f..405c8997078 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java index 4e51bc9e6c9..ea58501635b 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java @@ -47,6 +47,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; +import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -177,17 +178,28 @@ public class GroupByQueryEngine { if (dims.size() > 0) { List retVal = null; - for (Integer dimValue : dims.get(0).getRow()) { + List unaggregatedBuffers = null; + + final DimensionSelector dimSelector = dims.get(0); + final IndexedInts row = dimSelector.getRow(); + if (row.size() == 0) { ByteBuffer newKey = key.duplicate(); - newKey.putInt(dimValue); - final List unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); - if (unaggregatedBuffers != null) { - if (retVal == null) { - retVal = Lists.newArrayList(); - } - retVal.addAll(unaggregatedBuffers); + newKey.putInt(dimSelector.getValueCardinality()); + unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); + } + else { + for (Integer dimValue : row) { + ByteBuffer newKey = key.duplicate(); + newKey.putInt(dimValue); + unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); } } + if (unaggregatedBuffers != null) { + if (retVal == null) { + retVal = Lists.newArrayList(); + } + retVal.addAll(unaggregatedBuffers); + } return retVal; } else { @@ -385,7 +397,11 @@ public class GroupByQueryEngine ByteBuffer keyBuffer = input.getKey().duplicate(); for (int i = 0; i < dimensions.size(); ++i) { - theEvent.put(dimNames.get(i), dimensions.get(i).lookupName(keyBuffer.getInt())); + final DimensionSelector dimSelector = dimensions.get(i); + final int dimVal = keyBuffer.getInt(); + if (dimSelector.getValueCardinality() != dimVal) { + theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal)); + } } int position = input.getValue(); diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngineConfig.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngineConfig.java deleted file mode 100644 index fdd0dd84d37..00000000000 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngineConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012, 2013 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package io.druid.query.groupby; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class GroupByQueryEngineConfig -{ - @Config("druid.query.groupBy.maxIntermediateRows") - @Default("50000") - public abstract int getMaxIntermediateRows(); -} diff --git a/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java b/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java index 3e27c464f46..3585ce55db4 100644 --- a/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java @@ -38,6 +38,8 @@ public class LegacySegmentSpec extends MultipleIntervalSegmentSpec final List intervalStringList; if (intervals instanceof String) { intervalStringList = Arrays.asList((((String) intervals).split(","))); + } else if (intervals instanceof Interval) { + intervalStringList = Arrays.asList(intervals.toString()); } else if (intervals instanceof Map) { intervalStringList = (List) ((Map) intervals).get("intervals"); } else if (intervals instanceof List) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 1a2a5f9824b..fd43d81d698 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -292,7 +292,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter @Override public int getValueCardinality() { - return dimValLookup.size(); + return maxId; } @Override diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java new file mode 100644 index 00000000000..f4f089c92a4 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -0,0 +1,118 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.incremental; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.collections.StupidPool; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryEngine; +import junit.framework.Assert; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; + +/** + */ +public class IncrementalIndexStorageAdapterTest +{ + @Test + public void testSanity() throws Exception + { + IncrementalIndex index = new IncrementalIndex( + 0, QueryGranularity.MINUTE, new AggregatorFactory[]{new CountAggregatorFactory("cnt")} + ); + + index.add( + new MapBasedInputRow( + new DateTime().minus(1).getMillis(), + Lists.newArrayList("billy"), + ImmutableMap.of("billy", "hi") + ) + ); + index.add( + new MapBasedInputRow( + new DateTime().minus(1).getMillis(), + Lists.newArrayList("sally"), + ImmutableMap.of("sally", "bo") + ) + ); + + GroupByQueryEngine engine = new GroupByQueryEngine( + Suppliers.ofInstance(new GroupByQueryConfig() + { + + @Override + public int getMaxIntermediateRows() + { + return 5; + } + }), + new StupidPool( + new Supplier() + { + @Override + public ByteBuffer get() + { + return ByteBuffer.allocate(50000); + } + } + ) + ); + + final Sequence rows = engine.process( + GroupByQuery.builder() + .setDataSource("test") + .setGranularity(QueryGranularity.ALL) + .setInterval(new Interval(0, new DateTime().getMillis())) + .addDimension("billy") + .addDimension("sally") + .addAggregator(new LongSumAggregatorFactory("cnt", "cnt")) + .build(), + new IncrementalIndexStorageAdapter(index) + ); + + final ArrayList results = Sequences.toList(rows, Lists.newArrayList()); + + Assert.assertEquals(2, results.size()); + + MapBasedRow row = (MapBasedRow) results.get(0); + Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1l), row.getEvent()); + + row = (MapBasedRow) results.get(1); + Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1l), row.getEvent()); + } + +} diff --git a/realtime/pom.xml b/realtime/pom.xml index 950ae5bf83c..9d8e461c631 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java b/realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java index 984f1fb1b45..bddfaad8e38 100644 --- a/realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java +++ b/realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java @@ -39,6 +39,7 @@ import org.joda.time.DateTime; import java.io.IOException; import java.util.List; +import java.util.UUID; import java.util.concurrent.LinkedBlockingQueue; /** @@ -147,9 +148,7 @@ public class IrcFirehoseFactory implements FirehoseFactory @Override public List getAlternativeNicknames() { - return Lists.newArrayList(nick + "_", - nick + "__", - nick + "___"); + return Lists.newArrayList(nick + UUID.randomUUID(), nick + UUID.randomUUID(), nick + UUID.randomUUID()); } @Override diff --git a/realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java b/realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java index 5f2f5307dbe..f216b54ffac 100644 --- a/realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java +++ b/realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java @@ -50,8 +50,10 @@ class WikipediaIrcDecoder implements IrcDecoder final DatabaseReader geoLookup; static final Pattern pattern = Pattern.compile( - "\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03" + ".*\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)" + + "\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03.*" ); + static final Pattern ipPattern = Pattern.compile("\\d+.\\d+.\\d+.\\d+"); static final Pattern shortnamePattern = Pattern.compile("#(\\w\\w)\\..*"); diff --git a/realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java b/realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java index 64daf729484..d7ffb255063 100644 --- a/realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java +++ b/realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java @@ -30,7 +30,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; -import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.metamx.common.Pair; @@ -193,7 +192,7 @@ public class RealtimePlumberSchool implements PlumberSchool public void setQueryExecutorService(ExecutorService executorService) { - this.queryExecutorService = queryExecutorService; + this.queryExecutorService = executorService; } @Override diff --git a/server/pom.xml b/server/pom.xml index 39a1f46e48b..9d8b333e50d 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -29,7 +29,7 @@ com.metamx druid - 0.5.50-SNAPSHOT + 0.6.0-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 9a877f5007d..0f202976a75 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -1,7 +1,7 @@