Merge branch 'guice' of github.com:metamx/druid into guice

common/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

examples/bin/examples/wikipedia/query.body

@@ -12,8 +12,8 @@
       },
       {
          "type":"longSum",
-         "fieldName":"edits",
+         "fieldName":"edit_count",
-         "name":"edits"
+         "name":"count"
       }
    ],
    "filter":{

examples/bin/examples/wikipedia/wikipedia_realtime.spec

@@ -2,7 +2,7 @@
     "schema": {
         "dataSource": "wikipedia",
         "aggregators": [
-            {"type": "count", "name": "edits"},
+            {"type": "count", "name": "count"},
             {"type": "longSum", "fieldName": "added", "name": "added"},
             {"type": "longSum", "fieldName": "deleted", "name": "deleted"},
             {"type": "longSum", "fieldName": "delta", "name": "delta"}

examples/bin/run_example_client.sh

@@ -45,7 +45,7 @@ for delay in 5 30 30 30 30 30 30 30 30 30 30
    echo "sleep for $delay seconds..."
    echo " " 
    sleep $delay
-   curl -X POST 'http://localhost:8080/druid/v2/?w' -H 'content-type: application/json'  -d "`cat ${QUERY_FILE}`"
+   curl -X POST 'http://localhost:8083/druid/v2/?w' -H 'content-type: application/json'  -d "`cat ${QUERY_FILE}`"
    echo " "
    echo " "
 done

examples/bin/run_example_server.sh

@@ -58,7 +58,7 @@ DRUID_CP=${EXAMPLE_LOC}
 DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/../target/druid-examples-*-selfcontained.jar`
 DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/../config/realtime
 #For the kit
-DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/lib/druid-examples-*-selfcontained.jar`
+DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/lib/*
 DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/config/realtime
 
 echo "Running command:"

examples/config/realtime/runtime.properties

@@ -6,11 +6,16 @@ druid.service=realtime
 com.metamx.emitter.logging=true
 com.metamx.emitter.logging.level=info
 
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
 # processing
 druid.processing.buffer.sizeBytes=10000000
 
 # schema
-druid.realtime.specFile=realtime.spec
+druid.realtime.specFile=config/realtime/realtime.spec
 
 # aws (dummy user)
 com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ

examples/pom.xml

@@ -1,4 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Druid - a distributed column store.
+  ~ Copyright (C) 2012, 2013  Metamarkets Group Inc.
+  ~
+  ~ This program is free software; you can redistribute it and/or
+  ~ modify it under the terms of the GNU General Public License
+  ~ as published by the Free Software Foundation; either version 2
+  ~ of the License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+  -->
+
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.metamx.druid</groupId>
@@ -9,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

indexing-hadoop/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

indexing-service/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

pom.xml

@@ -23,7 +23,7 @@
     <groupId>com.metamx</groupId>
     <artifactId>druid</artifactId>
     <packaging>pom</packaging>
-    <version>0.5.50-SNAPSHOT</version>
+    <version>0.6.0-SNAPSHOT</version>
     <name>druid</name>
     <description>druid</description>
     <scm>
@@ -38,7 +38,7 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <metamx.java-util.version>0.25.0-SNAPSHOT</metamx.java-util.version>
+        <metamx.java-util.version>0.25.0</metamx.java-util.version>
         <apache.curator.version>2.1.0-incubating</apache.curator.version>
     </properties>
 
@@ -65,7 +65,7 @@
             <dependency>
                 <groupId>com.metamx</groupId>
                 <artifactId>emitter</artifactId>
-                <version>0.2.5-SNAPSHOT</version>
+                <version>0.2.5</version>
             </dependency>
             <dependency>
                 <groupId>com.metamx</groupId>
@@ -492,7 +492,7 @@
                 </plugin>
                 <plugin>
                     <artifactId>maven-shade-plugin</artifactId>
-                    <version>1.7.1</version>
+                    <version>2.1</version>
                 </plugin>
                 <plugin>
                     <artifactId>maven-site-plugin</artifactId>

processing/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java

@@ -47,6 +47,7 @@ import io.druid.query.dimension.DimensionSpec;
 import io.druid.segment.Cursor;
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.StorageAdapter;
+import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.Filters;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -177,17 +178,28 @@ public class GroupByQueryEngine
     {
       if (dims.size() > 0) {
         List<ByteBuffer> retVal = null;
-        for (Integer dimValue : dims.get(0).getRow()) {
+        List<ByteBuffer> unaggregatedBuffers = null;
+
+        final DimensionSelector dimSelector = dims.get(0);
+        final IndexedInts row = dimSelector.getRow();
+        if (row.size() == 0) {
+          ByteBuffer newKey = key.duplicate();
+          newKey.putInt(dimSelector.getValueCardinality());
+          unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
+        }
+        else {
+          for (Integer dimValue : row) {
             ByteBuffer newKey = key.duplicate();
             newKey.putInt(dimValue);
-          final List<ByteBuffer> unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
+            unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
+          }
+        }
         if (unaggregatedBuffers != null) {
           if (retVal == null) {
             retVal = Lists.newArrayList();
           }
           retVal.addAll(unaggregatedBuffers);
         }
-        }
         return retVal;
       }
       else {
@@ -385,7 +397,11 @@ public class GroupByQueryEngine
 
                   ByteBuffer keyBuffer = input.getKey().duplicate();
                   for (int i = 0; i < dimensions.size(); ++i) {
-                    theEvent.put(dimNames.get(i), dimensions.get(i).lookupName(keyBuffer.getInt()));
+                    final DimensionSelector dimSelector = dimensions.get(i);
+                    final int dimVal = keyBuffer.getInt();
+                    if (dimSelector.getValueCardinality() != dimVal) {
+                      theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal));
+                    }
                   }
 
                   int position = input.getValue();

processing/src/main/java/io/druid/query/groupby/GroupByQueryEngineConfig.java

@@ -1,32 +0,0 @@
-/*
- * Druid - a distributed column store.
- * Copyright (C) 2012, 2013  Metamarkets Group Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package io.druid.query.groupby;
-
-import org.skife.config.Config;
-import org.skife.config.Default;
-
-/**
- */
-public abstract class GroupByQueryEngineConfig
-{
-  @Config("druid.query.groupBy.maxIntermediateRows")
-  @Default("50000")
-  public abstract int getMaxIntermediateRows();
-}

processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java

@@ -38,6 +38,8 @@ public class LegacySegmentSpec extends MultipleIntervalSegmentSpec
     final List<?> intervalStringList;
     if (intervals instanceof String) {
       intervalStringList = Arrays.asList((((String) intervals).split(",")));
+    } else if (intervals instanceof Interval) {
+      intervalStringList = Arrays.asList(intervals.toString());
     } else if (intervals instanceof Map) {
       intervalStringList = (List) ((Map) intervals).get("intervals");
     } else if (intervals instanceof List) {

processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java

@@ -292,7 +292,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
                           @Override
                           public int getValueCardinality()
                           {
-                            return dimValLookup.size();
+                            return maxId;
                           }
 
                           @Override

processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java

@@ -0,0 +1,118 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.segment.incremental;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.metamx.common.guava.Sequence;
+import com.metamx.common.guava.Sequences;
+import io.druid.collections.StupidPool;
+import io.druid.data.input.MapBasedInputRow;
+import io.druid.data.input.MapBasedRow;
+import io.druid.data.input.Row;
+import io.druid.granularity.QueryGranularity;
+import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.query.aggregation.CountAggregatorFactory;
+import io.druid.query.aggregation.LongSumAggregatorFactory;
+import io.druid.query.groupby.GroupByQuery;
+import io.druid.query.groupby.GroupByQueryConfig;
+import io.druid.query.groupby.GroupByQueryEngine;
+import junit.framework.Assert;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+/**
+ */
+public class IncrementalIndexStorageAdapterTest
+{
+  @Test
+  public void testSanity() throws Exception
+  {
+    IncrementalIndex index = new IncrementalIndex(
+        0, QueryGranularity.MINUTE, new AggregatorFactory[]{new CountAggregatorFactory("cnt")}
+    );
+
+    index.add(
+        new MapBasedInputRow(
+            new DateTime().minus(1).getMillis(),
+            Lists.newArrayList("billy"),
+            ImmutableMap.<String, Object>of("billy", "hi")
+        )
+    );
+    index.add(
+        new MapBasedInputRow(
+            new DateTime().minus(1).getMillis(),
+            Lists.newArrayList("sally"),
+            ImmutableMap.<String, Object>of("sally", "bo")
+        )
+    );
+
+    GroupByQueryEngine engine = new GroupByQueryEngine(
+        Suppliers.<GroupByQueryConfig>ofInstance(new GroupByQueryConfig()
+        {
+
+          @Override
+          public int getMaxIntermediateRows()
+          {
+            return 5;
+          }
+        }),
+        new StupidPool(
+            new Supplier<ByteBuffer>()
+            {
+              @Override
+              public ByteBuffer get()
+              {
+                return ByteBuffer.allocate(50000);
+              }
+            }
+        )
+    );
+
+    final Sequence<Row> rows = engine.process(
+        GroupByQuery.builder()
+                    .setDataSource("test")
+                    .setGranularity(QueryGranularity.ALL)
+                    .setInterval(new Interval(0, new DateTime().getMillis()))
+                    .addDimension("billy")
+                    .addDimension("sally")
+                    .addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
+                    .build(),
+        new IncrementalIndexStorageAdapter(index)
+    );
+
+    final ArrayList<Row> results = Sequences.toList(rows, Lists.<Row>newArrayList());
+
+    Assert.assertEquals(2, results.size());
+
+    MapBasedRow row = (MapBasedRow) results.get(0);
+    Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1l), row.getEvent());
+
+    row = (MapBasedRow) results.get(1);
+    Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1l), row.getEvent());
+  }
+
+}

realtime/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java

@@ -39,6 +39,7 @@ import org.joda.time.DateTime;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.LinkedBlockingQueue;
 
 /**
@@ -147,9 +148,7 @@ public class IrcFirehoseFactory implements FirehoseFactory
           @Override
           public List<String> getAlternativeNicknames()
           {
-            return Lists.newArrayList(nick + "_",
+            return Lists.newArrayList(nick + UUID.randomUUID(), nick + UUID.randomUUID(), nick + UUID.randomUUID());
-                                      nick + "__",
-                                      nick + "___");
           }
 
           @Override

realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java

@@ -50,8 +50,10 @@ class WikipediaIrcDecoder implements IrcDecoder
   final DatabaseReader geoLookup;
 
   static final Pattern pattern = Pattern.compile(
-      "\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03"
+      ".*\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)" +
+      "\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03.*"
   );
+
   static final Pattern ipPattern = Pattern.compile("\\d+.\\d+.\\d+.\\d+");
   static final Pattern shortnamePattern = Pattern.compile("#(\\w\\w)\\..*");
 

realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java

@@ -30,7 +30,6 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Ints;
-import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.metamx.common.Pair;
@@ -193,7 +192,7 @@ public class RealtimePlumberSchool implements PlumberSchool
 
   public void setQueryExecutorService(ExecutorService executorService)
   {
-    this.queryExecutorService = queryExecutorService;
+    this.queryExecutorService = executorService;
   }
 
   @Override

server/pom.xml

@@ -29,7 +29,7 @@
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>

services/pom.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
   ~ Druid - a distributed column store.
-  ~ Copyright (C) 2012  Metamarkets Group Inc.
+  ~ Copyright (C) 2012, 2013  Metamarkets Group Inc.
   ~
   ~ This program is free software; you can redistribute it and/or
   ~ modify it under the terms of the GNU General Public License
@@ -24,14 +24,29 @@
     <artifactId>druid-services</artifactId>
     <name>druid-services</name>
     <description>druid-services</description>
-    <version>0.5.50-SNAPSHOT</version>
+    <version>0.6.0-SNAPSHOT</version>
     <parent>
         <groupId>com.metamx</groupId>
         <artifactId>druid</artifactId>
-        <version>0.5.50-SNAPSHOT</version>
+        <version>0.6.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>
+        <dependency>
+            <groupId>com.metamx.druid</groupId>
+            <artifactId>druid-examples</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.metamx.druid</groupId>
+            <artifactId>druid-indexing-hadoop</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.metamx.druid</groupId>
+            <artifactId>druid-indexing-service</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.metamx.druid</groupId>
             <artifactId>druid-realtime</artifactId>

services/src/assembly/assembly.xml

@@ -1,5 +1,7 @@
 <?xml version="1.0"?>
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
     <id>bin</id>
     <formats>
         <format>tar.gz</format>
@@ -48,20 +50,6 @@
             <fileMode>744</fileMode>
             <outputDirectory>/</outputDirectory>
         </fileSet>
-        <fileSet>
-            <directory>../examples/target</directory>
-            <includes>
-                <include>druid-examples-*-selfcontained.jar</include>
-            </includes>
-            <outputDirectory>lib</outputDirectory>
-        </fileSet>
-        <fileSet>
-            <directory>../services/target</directory>
-            <includes>
-                <include>druid-services-*-selfcontained.jar</include>
-            </includes>
-            <outputDirectory>lib</outputDirectory>
-        </fileSet>
         <fileSet>
             <directory>../examples/bin/examples</directory>
             <includes>