Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'guice' of github.com:metamx/druid into guice

This commit is contained in:
fjy 2013-09-18 15:59:29 -07:00
parent b3bfcb7792 e019dddaaa
commit 8d7af40313
23 changed files with 214 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

4
examples/bin/examples/wikipedia/query.body
View File

@ -12,8 +12,8 @@
},
{
"type":"longSum",
"fieldName":"edits",
"name":"edits"
"fieldName":"edit_count",
"name":"count"
}
],
"filter":{

2
examples/bin/examples/wikipedia/wikipedia_realtime.spec
View File

@ -2,7 +2,7 @@
"schema": {
"dataSource": "wikipedia",
"aggregators": [
{"type": "count", "name": "edits"},
{"type": "count", "name": "count"},
{"type": "longSum", "fieldName": "added", "name": "added"},
{"type": "longSum", "fieldName": "deleted", "name": "deleted"},
{"type": "longSum", "fieldName": "delta", "name": "delta"}

2
examples/bin/run_example_client.sh
View File

@ -45,7 +45,7 @@ for delay in 5 30 30 30 30 30 30 30 30 30 30
echo "sleep for $delay seconds..."
echo " "
sleep $delay
curl -X POST 'http://localhost:8080/druid/v2/?w' -H 'content-type: application/json' -d "`cat ${QUERY_FILE}`"
curl -X POST 'http://localhost:8083/druid/v2/?w' -H 'content-type: application/json' -d "`cat ${QUERY_FILE}`"
echo " "
echo " "
done

2
examples/bin/run_example_server.sh
View File

@ -58,7 +58,7 @@ DRUID_CP=${EXAMPLE_LOC}
DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/../target/druid-examples-*-selfcontained.jar`
DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/../config/realtime
#For the kit
DRUID_CP=${DRUID_CP}:`ls ${SCRIPT_DIR}/lib/druid-examples-*-selfcontained.jar`
DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/lib/*
DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/config/realtime
echo "Running command:"

7
examples/config/realtime/runtime.properties
View File

@ -6,11 +6,16 @@ druid.service=realtime
com.metamx.emitter.logging=true
com.metamx.emitter.logging.level=info
# zk
druid.zk.service.host=localhost
druid.zk.paths.base=/druid
druid.zk.paths.discoveryPath=/druid/discoveryPath
# processing
druid.processing.buffer.sizeBytes=10000000
# schema
druid.realtime.specFile=realtime.spec
druid.realtime.specFile=config/realtime/realtime.spec
# aws (dummy user)
com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ

21
examples/pom.xml
View File

@ -1,4 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Druid - a distributed column store.
~ Copyright (C) 2012, 2013 Metamarkets Group Inc.
~
~ This program is free software; you can redistribute it and/or
~ modify it under the terms of the GNU General Public License
~ as published by the Free Software Foundation; either version 2
~ of the License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program; if not, write to the Free Software
~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.metamx.druid</groupId>
@ -9,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

2
indexing-hadoop/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

2
indexing-service/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

8
pom.xml
View File

@ -23,7 +23,7 @@
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<packaging>pom</packaging>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<name>druid</name>
<description>druid</description>
<scm>
@ -38,7 +38,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<metamx.java-util.version>0.25.0-SNAPSHOT</metamx.java-util.version>
<metamx.java-util.version>0.25.0</metamx.java-util.version>
<apache.curator.version>2.1.0-incubating</apache.curator.version>
</properties>
@ -65,7 +65,7 @@
<dependency>
<groupId>com.metamx</groupId>
<artifactId>emitter</artifactId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</dependency>
<dependency>
<groupId>com.metamx</groupId>
@ -492,7 +492,7 @@
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>1.7.1</version>
<version>2.1</version>
</plugin>
<plugin>
<artifactId>maven-site-plugin</artifactId>

2
processing/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

34
processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java
View File

@ -47,6 +47,7 @@ import io.druid.query.dimension.DimensionSpec;
import io.druid.segment.Cursor;
import io.druid.segment.DimensionSelector;
import io.druid.segment.StorageAdapter;
import io.druid.segment.data.IndexedInts;
import io.druid.segment.filter.Filters;
import org.joda.time.DateTime;
import org.joda.time.Interval;
@ -177,17 +178,28 @@ public class GroupByQueryEngine
{
if (dims.size() > 0) {
List<ByteBuffer> retVal = null;
for (Integer dimValue : dims.get(0).getRow()) {
List<ByteBuffer> unaggregatedBuffers = null;
final DimensionSelector dimSelector = dims.get(0);
final IndexedInts row = dimSelector.getRow();
if (row.size() == 0) {
ByteBuffer newKey = key.duplicate();
newKey.putInt(dimValue);
final List<ByteBuffer> unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
if (unaggregatedBuffers != null) {
if (retVal == null) {
retVal = Lists.newArrayList();
}
retVal.addAll(unaggregatedBuffers);
newKey.putInt(dimSelector.getValueCardinality());
unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
}
else {
for (Integer dimValue : row) {
ByteBuffer newKey = key.duplicate();
newKey.putInt(dimValue);
unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
}
}
if (unaggregatedBuffers != null) {
if (retVal == null) {
retVal = Lists.newArrayList();
}
retVal.addAll(unaggregatedBuffers);
}
return retVal;
}
else {
@ -385,7 +397,11 @@ public class GroupByQueryEngine
ByteBuffer keyBuffer = input.getKey().duplicate();
for (int i = 0; i < dimensions.size(); ++i) {
theEvent.put(dimNames.get(i), dimensions.get(i).lookupName(keyBuffer.getInt()));
final DimensionSelector dimSelector = dimensions.get(i);
final int dimVal = keyBuffer.getInt();
if (dimSelector.getValueCardinality() != dimVal) {
theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal));
}
}
int position = input.getValue();

32
processing/src/main/java/io/druid/query/groupby/GroupByQueryEngineConfig.java
View File

@ -1,32 +0,0 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.query.groupby;
import org.skife.config.Config;
import org.skife.config.Default;
/**
*/
public abstract class GroupByQueryEngineConfig
{
@Config("druid.query.groupBy.maxIntermediateRows")
@Default("50000")
public abstract int getMaxIntermediateRows();
}

2
processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java
View File

@ -38,6 +38,8 @@ public class LegacySegmentSpec extends MultipleIntervalSegmentSpec
final List<?> intervalStringList;
if (intervals instanceof String) {
intervalStringList = Arrays.asList((((String) intervals).split(",")));
} else if (intervals instanceof Interval) {
intervalStringList = Arrays.asList(intervals.toString());
} else if (intervals instanceof Map) {
intervalStringList = (List) ((Map) intervals).get("intervals");
} else if (intervals instanceof List) {

2
processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
View File

@ -292,7 +292,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
@Override
public int getValueCardinality()
{
return dimValLookup.size();
return maxId;
}
@Override

118
processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.segment.incremental;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.metamx.common.guava.Sequence;
import com.metamx.common.guava.Sequences;
import io.druid.collections.StupidPool;
import io.druid.data.input.MapBasedInputRow;
import io.druid.data.input.MapBasedRow;
import io.druid.data.input.Row;
import io.druid.granularity.QueryGranularity;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.CountAggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
import io.druid.query.groupby.GroupByQuery;
import io.druid.query.groupby.GroupByQueryConfig;
import io.druid.query.groupby.GroupByQueryEngine;
import junit.framework.Assert;
import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.junit.Test;
import java.nio.ByteBuffer;
import java.util.ArrayList;
/**
*/
public class IncrementalIndexStorageAdapterTest
{
@Test
public void testSanity() throws Exception
{
IncrementalIndex index = new IncrementalIndex(
0, QueryGranularity.MINUTE, new AggregatorFactory[]{new CountAggregatorFactory("cnt")}
);
index.add(
new MapBasedInputRow(
new DateTime().minus(1).getMillis(),
Lists.newArrayList("billy"),
ImmutableMap.<String, Object>of("billy", "hi")
)
);
index.add(
new MapBasedInputRow(
new DateTime().minus(1).getMillis(),
Lists.newArrayList("sally"),
ImmutableMap.<String, Object>of("sally", "bo")
)
);
GroupByQueryEngine engine = new GroupByQueryEngine(
Suppliers.<GroupByQueryConfig>ofInstance(new GroupByQueryConfig()
{
@Override
public int getMaxIntermediateRows()
{
return 5;
}
}),
new StupidPool(
new Supplier<ByteBuffer>()
{
@Override
public ByteBuffer get()
{
return ByteBuffer.allocate(50000);
}
}
)
);
final Sequence<Row> rows = engine.process(
GroupByQuery.builder()
.setDataSource("test")
.setGranularity(QueryGranularity.ALL)
.setInterval(new Interval(0, new DateTime().getMillis()))
.addDimension("billy")
.addDimension("sally")
.addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
.build(),
new IncrementalIndexStorageAdapter(index)
);
final ArrayList<Row> results = Sequences.toList(rows, Lists.<Row>newArrayList());
Assert.assertEquals(2, results.size());
MapBasedRow row = (MapBasedRow) results.get(0);
Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1l), row.getEvent());
row = (MapBasedRow) results.get(1);
Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1l), row.getEvent());
}
}

2
realtime/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

5
realtime/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java
View File

@ -39,6 +39,7 @@ import org.joda.time.DateTime;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.LinkedBlockingQueue;
/**
@ -147,9 +148,7 @@ public class IrcFirehoseFactory implements FirehoseFactory
@Override
public List<String> getAlternativeNicknames()
{
return Lists.newArrayList(nick + "_",
nick + "__",
nick + "___");
return Lists.newArrayList(nick + UUID.randomUUID(), nick + UUID.randomUUID(), nick + UUID.randomUUID());
}
@Override

4
realtime/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java
View File

@ -50,8 +50,10 @@ class WikipediaIrcDecoder implements IrcDecoder
final DatabaseReader geoLookup;
static final Pattern pattern = Pattern.compile(
"\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03"
".*\\x0314\\[\\[\\x0307(.+?)\\x0314\\]\\]\\x034 (.*?)\\x0310.*\\x0302(http.+?)" +
"\\x03.+\\x0303(.+?)\\x03.+\\x03 (\\(([+-]\\d+)\\).*|.+) \\x0310(.+)\\x03.*"
);
static final Pattern ipPattern = Pattern.compile("\\d+.\\d+.\\d+.\\d+");
static final Pattern shortnamePattern = Pattern.compile("#(\\w\\w)\\..*");

3
realtime/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java
View File

@ -30,7 +30,6 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.metamx.common.Pair;
@ -193,7 +192,7 @@ public class RealtimePlumberSchool implements PlumberSchool
public void setQueryExecutorService(ExecutorService executorService)
{
this.queryExecutorService = queryExecutorService;
this.queryExecutorService = executorService;
}
@Override

2
server/pom.xml
View File

@ -29,7 +29,7 @@
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>

21
services/pom.xml
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Druid - a distributed column store.
~ Copyright (C) 2012 Metamarkets Group Inc.
~ Copyright (C) 2012, 2013 Metamarkets Group Inc.
~
~ This program is free software; you can redistribute it and/or
~ modify it under the terms of the GNU General Public License
@ -24,14 +24,29 @@
<artifactId>druid-services</artifactId>
<name>druid-services</name>
<description>druid-services</description>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<parent>
<groupId>com.metamx</groupId>
<artifactId>druid</artifactId>
<version>0.5.50-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.metamx.druid</groupId>
<artifactId>druid-examples</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.metamx.druid</groupId>
<artifactId>druid-indexing-hadoop</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.metamx.druid</groupId>
<artifactId>druid-indexing-service</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.metamx.druid</groupId>
<artifactId>druid-realtime</artifactId>

18
services/src/assembly/assembly.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0"?>
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>bin</id>
<formats>
<format>tar.gz</format>
@ -48,20 +50,6 @@
<fileMode>744</fileMode>
<outputDirectory>/</outputDirectory>
</fileSet>
<fileSet>
<directory>../examples/target</directory>
<includes>
<include>druid-examples-*-selfcontained.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
</fileSet>
<fileSet>
<directory>../services/target</directory>
<includes>
<include>druid-services-*-selfcontained.jar</include>
</includes>
<outputDirectory>lib</outputDirectory>
</fileSet>
<fileSet>
<directory>../examples/bin/examples</directory>
<includes>