Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fjy

This commit is contained in:
Fangjin Yang 2012-11-19 15:07:00 -08:00
parent b228c053e5 c657969243
commit 4131063049
55 changed files with 864 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/main/java/com/metamx/druid/query/dimension/DefaultDimensionSpec.java
View File

@ -39,7 +39,7 @@ public class DefaultDimensionSpec implements DimensionSpec
@JsonProperty("outputName") String outputName
)
{
this.dimension = dimension.toLowerCase();
this.dimension = dimension;
// Do null check for legacy backwards compatibility, callers should be setting the value.
this.outputName = outputName == null ? dimension : outputName;

2
client/src/main/java/com/metamx/druid/query/dimension/ExtractionDimensionSpec.java
View File

@ -42,7 +42,7 @@ public class ExtractionDimensionSpec implements DimensionSpec
@JsonProperty("dimExtractionFn") DimExtractionFn dimExtractionFn
)
{
this.dimension = dimension.toLowerCase();
this.dimension = dimension;
this.dimExtractionFn = dimExtractionFn;
// Do null check for backwards compatibility

2
client/src/main/java/com/metamx/druid/query/filter/ExtractionDimFilter.java
View File

@ -42,7 +42,7 @@ public class ExtractionDimFilter implements DimFilter
@JsonProperty("dimExtractionFn") DimExtractionFn dimExtractionFn
)
{
this.dimension = dimension.toLowerCase();
this.dimension = dimension;
this.value = value;
this.dimExtractionFn = dimExtractionFn;
}

2
client/src/main/java/com/metamx/druid/query/filter/SelectorDimFilter.java
View File

@ -39,7 +39,7 @@ public class SelectorDimFilter implements DimFilter
@JsonProperty("value") String value
)
{
this.dimension = dimension.toLowerCase();
this.dimension = dimension;
this.value = value;
}

2
client/src/main/java/com/metamx/druid/query/search/SearchQuery.java
View File

@ -69,7 +69,7 @@ public class SearchQuery extends BaseQuery<Result<SearchResultValue>>
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
return input;
}
}
);

2
common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregatorFactory.java
View File

@ -45,7 +45,7 @@ public class DoubleSumAggregatorFactory implements AggregatorFactory
)
{
this.name = name;
this.fieldName = fieldName.toLowerCase();
this.fieldName = fieldName;
}
@Override

2
common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregatorFactory.java
View File

@ -57,7 +57,7 @@ public class JavaScriptAggregatorFactory implements AggregatorFactory
)
{
this.name = name;
this.script = expression.toLowerCase();
this.script = expression;
this.fieldNames = fieldNames;
this.combiner = compileScript(script);
}

2
common/src/main/java/com/metamx/druid/aggregation/LongSumAggregatorFactory.java
View File

@ -45,7 +45,7 @@ public class LongSumAggregatorFactory implements AggregatorFactory
)
{
this.name = name;
this.fieldName = fieldName.toLowerCase();
this.fieldName = fieldName;
}
@Override

2
common/src/main/java/com/metamx/druid/aggregation/MaxAggregatorFactory.java
View File

@ -45,7 +45,7 @@ public class MaxAggregatorFactory implements AggregatorFactory
)
{
this.name = name;
this.fieldName = fieldName.toLowerCase();
this.fieldName = fieldName;
}
@Override

2
common/src/main/java/com/metamx/druid/aggregation/MinAggregatorFactory.java
View File

@ -45,7 +45,7 @@ public class MinAggregatorFactory implements AggregatorFactory
)
{
this.name = name;
this.fieldName = fieldName.toLowerCase();
this.fieldName = fieldName;
}
@Override

17
common/src/main/java/com/metamx/druid/input/InputRow.java
View File

@ -22,8 +22,25 @@ package com.metamx.druid.input;
import java.util.List;
/**
* An InputRow is the interface definition of an event being input into the data ingestion layer.
*
* An InputRow is a Row with a self-describing list of the dimensions available. This list is used to
* implement "schema-less" data ingestion that allows the system to add new dimensions as they appear.
*
* Note, Druid is a case-insensitive system for parts of schema (column names), this has direct implications
* for the implementation of InputRows and Rows. The case-insensitiveness is implemented by lowercasing all
* schema elements before looking them up, this means that calls to getDimension() and getFloatMetric() will
* have all lowercase column names passed in no matter what is returned from getDimensions or passed in as the
* fieldName of an AggregatorFactory. Implementations of InputRow and Row should expect to get values back
* in all lowercase form (i.e. they should either have already turned everything into lowercase or they
* should operate in a case-insensitive manner).
*/
public interface InputRow extends Row
{
/**
* Returns the dimensions that exist in this row.
*
* @return the dimensions that exist in this row.
*/
public List<String> getDimensions();
}

34
common/src/main/java/com/metamx/druid/input/Row.java
View File

@ -25,6 +25,16 @@ import org.codehaus.jackson.annotate.JsonTypeInfo;
import java.util.List;
/**
* A Row of data. This can be used for both input and output into various parts of the system. It assumes
* that the user already knows the schema of the row and can query for the parts that they care about.
*
* Note, Druid is a case-insensitive system for parts of schema (column names), this has direct implications
* for the implementation of InputRows and Rows. The case-insensitiveness is implemented by lowercasing all
* schema elements before looking them up, this means that calls to getDimension() and getFloatMetric() will
* have all lowercase column names passed in no matter what is returned from getDimensions or passed in as the
* fieldName of an AggregatorFactory. Implementations of InputRow and Row should expect to get values back
* in all lowercase form (i.e. they should either have already turned everything into lowercase or they
* should operate in a case-insensitive manner).
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "version")
@JsonSubTypes(value = {
@ -32,7 +42,31 @@ import java.util.List;
})
public interface Row
{
/**
* Returns the timestamp from the epoch in milliseconds. If the event happened _right now_, this would return the
* same thing as System.currentTimeMillis();
*
* @return the timestamp from the epoch in milliseconds.
*/
public long getTimestampFromEpoch();
/**
* Returns the list of dimension values for the given column name.
*
* Column names are always all lowercase in order to support case-insensitive schemas.
*
* @param dimension the lowercase column name of the dimension requested
* @return the list of values for the provided column name
*/
public List<String> getDimension(String dimension);
/**
* Returns the float value of the given metric column.
*
* Column names are always all lowercase in order to support case-insensitive schemas.
*
* @param metric the lowercase column name of the metric requested
* @return the float value for the provided column name.
*/
public float getFloatMetric(String metric);
}

39
doc/data_flow_realtime.dot Normal file
View File

@ -0,0 +1,39 @@
digraph DruidDataFlow {
graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB];
node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"];
edge [color="#377d18"];
1 [label = "REST query"];
2 [label = "realtime_data_src"];
4 [label = "indexed segments\nblob_store (S3, HDFS)"];
5 [label = "Realtime.working_set"];
6 [label = "Realtime.spill_disk"];
7 [label = "HDFS_data_src"];
8 [label = "external_data_src"];
9 [label = "Realtime.local_disk"];
10 [label = "metadata_store (mysql)"];
11 [label = "HadoopDruidIndexer.working_set"];
12 [label = "IndexerService.working_set"];
2 -> 5 [label = "Realtime.Firehose"];
5 -> 6 [label = "Realtime.Firehose"];
5 -> 9 [label = "Realtime.Indexer"];
6 -> 9 [label = "Realtime.Indexer"];
5 -> 10 [label = "Realtime.MetadataUpdater"];
7 -> 11 [label = "HadoopDruidIndexer"];
11 -> 4 [label = "HadoopDruidIndexer"];
11 -> 10 [label = "HadoopDruidIndexer"];
8 -> 12 [label = "IndexerService"];
12 -> 4 [label = "IndexerService"];
12 -> 10 [label = "IndexerService"];
9 -> 4 [label = "Realtime.segmentPusher"];
5 -> 1 [label = " results "];
1 -> 5 [label = " query "];
}

BIN
doc/data_flow_realtime.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 96 KiB

128
doc/data_flow_realtime.svg Normal file
View File

@ -0,0 +1,128 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<!-- Generated by dot version 1.13 (v16) (Mon August 23, 2004)
For user: (peb) Paul E Baclace Title: DruidDataFlow Pages: 1 -->
<svg width="950pt" height="404pt"
viewBox = "-1 -1 949 403"
xmlns="http://www.w3.org/2000/svg">
<g id="graph0" class="graph" style="font-family:Times-Roman;font-size:14.00;">
<title>DruidDataFlow</title>
<polygon style="fill:#fffef5;stroke:#fffef5;" points="0,401 0,-1 948,-1 948,401 0,401"/>
<g id="node1" class="node"><title>1</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="177,4 91,4 91,40 177,40 177,4"/>
<text text-anchor="middle" x="134" y="27" style="font-family:Helvetica;">REST query</text>
</g>
<g id="node4" class="node"><title>5</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="272,92 124,92 124,128 272,128 272,92"/>
<text text-anchor="middle" x="198" y="115" style="font-family:Helvetica;">Realtime.working_set</text>
</g>
<g id="edge28" class="edge"><title>1-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M162,40C169,45 175,51 180,58 185,65 189,74 192,82"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="195,82 194,92 189,83 195,82"/>
<text text-anchor="middle" x="205" y="71"> query </text>
</g>
<g id="node2" class="node"><title>2</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="335,4 209,4 209,40 335,40 335,4"/>
<text text-anchor="middle" x="272" y="27" style="font-family:Helvetica;">realtime_data_src</text>
</g>
<g id="edge2" class="edge"><title>2-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M257,40C246,53 232,70 220,84"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="222,87 213,92 217,82 222,87"/>
<text text-anchor="middle" x="293" y="71">Realtime.Firehose</text>
</g>
<g id="node3" class="node"><title>4</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="328,356 172,356 172,396 328,396 328,356"/>
<text text-anchor="middle" x="250" y="373" style="font-family:Helvetica;">indexed segments</text>
<text text-anchor="middle" x="250" y="389" style="font-family:Helvetica;">blob_store (S3, HDFS)</text>
</g>
<g id="edge26" class="edge"><title>5-&gt;1</title>
<path style="fill:none;stroke:#377d18;" d="M150,92C143,87 137,81 133,74 129,67 128,58 128,50"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="125,50 129,40 131,50 125,50"/>
<text text-anchor="middle" x="154" y="71"> results </text>
</g>
<g id="node5" class="node"><title>6</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="136,180 6,180 6,216 136,216 136,180"/>
<text text-anchor="middle" x="71" y="203" style="font-family:Helvetica;">Realtime.spill_disk</text>
</g>
<g id="edge4" class="edge"><title>5-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M123,127C105,133 89,139 82,146 76,152 74,161 72,170"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="75,170 71,180 69,170 75,170"/>
<text text-anchor="middle" x="136" y="159">Realtime.Firehose</text>
</g>
<g id="node8" class="node"><title>9</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="143,268 9,268 9,304 143,304 143,268"/>
<text text-anchor="middle" x="76" y="291" style="font-family:Helvetica;">Realtime.local_disk</text>
</g>
<g id="edge6" class="edge"><title>5-&gt;9</title>
<path style="fill:none;stroke:#377d18;" d="M200,128C203,158 204,215 175,250 171,255 162,260 151,264"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="153,267 142,268 150,261 153,267"/>
<text text-anchor="middle" x="247" y="203">Realtime.Indexer</text>
</g>
<g id="node9" class="node"><title>10</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="726,358 564,358 564,394 726,394 726,358"/>
<text text-anchor="middle" x="645" y="381" style="font-family:Helvetica;">metadata_store (mysql)</text>
</g>
<g id="edge10" class="edge"><title>5-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M273,115C391,125 611,146 682,180 751,214 798,276 754,338 750,344 739,349 727,354"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="728,358 717,358 725,351 728,358"/>
<text text-anchor="middle" x="835" y="247">Realtime.MetadataUpdater</text>
</g>
<g id="edge8" class="edge"><title>6-&gt;9</title>
<path style="fill:none;stroke:#377d18;" d="M72,216C73,228 74,244 74,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="77,258 75,268 71,258 77,258"/>
<text text-anchor="middle" x="123" y="247">Realtime.Indexer</text>
</g>
<g id="node6" class="node"><title>7</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="672,180 558,180 558,216 672,216 672,180"/>
<text text-anchor="middle" x="615" y="203" style="font-family:Helvetica;">HDFS_data_src</text>
</g>
<g id="node10" class="node"><title>11</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="726,268 504,268 504,304 726,304 726,268"/>
<text text-anchor="middle" x="615" y="291" style="font-family:Helvetica;">HadoopDruidIndexer.working_set</text>
</g>
<g id="edge12" class="edge"><title>7-&gt;11</title>
<path style="fill:none;stroke:#377d18;" d="M615,216C615,228 615,244 615,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="619,258 615,268 612,258 619,258"/>
<text text-anchor="middle" x="674" y="247">HadoopDruidIndexer</text>
</g>
<g id="node7" class="node"><title>8</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="438,180 312,180 312,216 438,216 438,180"/>
<text text-anchor="middle" x="375" y="203" style="font-family:Helvetica;">external_data_src</text>
</g>
<g id="node11" class="node"><title>12</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="464,268 276,268 276,304 464,304 464,268"/>
<text text-anchor="middle" x="370" y="291" style="font-family:Helvetica;">IndexerService.working_set</text>
</g>
<g id="edge18" class="edge"><title>8-&gt;12</title>
<path style="fill:none;stroke:#377d18;" d="M374,216C373,228 372,244 372,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="375,258 371,268 369,258 375,258"/>
<text text-anchor="middle" x="414" y="247">IndexerService</text>
</g>
<g id="edge24" class="edge"><title>9-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M85,304C91,316 100,330 112,338 126,348 144,356 161,361"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="162,358 171,364 160,364 162,358"/>
<text text-anchor="middle" x="181" y="335">Realtime.segmentPusher</text>
</g>
<g id="edge14" class="edge"><title>11-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M553,304C536,309 519,315 503,322 488,328 486,334 472,338 430,353 380,362 339,368"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="339,371 329,369 339,365 339,371"/>
<text text-anchor="middle" x="568" y="335">HadoopDruidIndexer</text>
</g>
<g id="edge16" class="edge"><title>11-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M621,304C625,317 631,334 636,348"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="639,347 639,358 633,349 639,347"/>
<text text-anchor="middle" x="690" y="335">HadoopDruidIndexer</text>
</g>
<g id="edge20" class="edge"><title>12-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M300,304C284,310 270,316 264,322 258,328 255,337 253,346"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="256,347 251,356 250,346 256,347"/>
<text text-anchor="middle" x="306" y="335">IndexerService</text>
</g>
<g id="edge22" class="edge"><title>12-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M370,304C371,316 374,330 384,338 396,349 483,361 554,368"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="554,365 564,369 554,371 554,365"/>
<text text-anchor="middle" x="426" y="335">IndexerService</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 7.5 KiB

27
doc/data_flow_simple.dot Normal file
View File

@ -0,0 +1,27 @@
digraph DruidDataFlow {
graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB];
node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"];
edge [color="#377d18"];
1 [label = "REST query"];
10 [label = "Broker.query_cache"];
2 [label = "realtime_data_src"];
3 [label = "Compute.disk_cache"];
4 [label = "indexed segments\nblob_store (S3, HDFS)"];
5 [label = "Realtime.working_set"];
6 [label = "Compute.working_set"];
2 -> 5 [label = "Realtime.Firehose"];
5 -> 4 [label = "Realtime.segmentPusher"];
4 -> 3 [label = "Compute.load"];
3 -> 6 [label = "Compute.map"];
5 -> 10 [label = " results "];
6 -> 10 [label = " results "];
10 -> 1 [label = " results "];
10 -> 5 [label = " query "];
10 -> 6 [label = " query "];
1 -> 10 [label = " query "];
}

BIN
doc/data_flow_simple.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

92
doc/data_flow_simple.svg Normal file
View File

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<!-- Generated by dot version 1.13 (v16) (Mon August 23, 2004)
For user: (peb) Paul E Baclace Title: DruidDataFlow Pages: 1 -->
<svg width="405pt" height="492pt"
viewBox = "-1 -1 404 491"
xmlns="http://www.w3.org/2000/svg">
<g id="graph0" class="graph" style="font-family:Times-Roman;font-size:14.00;">
<title>DruidDataFlow</title>
<polygon style="fill:#fffef5;stroke:#fffef5;" points="0,489 0,-1 403,-1 403,489 0,489"/>
<g id="node1" class="node"><title>1</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="163,4 77,4 77,40 163,40 163,4"/>
<text text-anchor="middle" x="120" y="27" style="font-family:Helvetica;">REST query</text>
</g>
<g id="node2" class="node"><title>10</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="189,92 51,92 51,128 189,128 189,92"/>
<text text-anchor="middle" x="120" y="115" style="font-family:Helvetica;">Broker.query_cache</text>
</g>
<g id="edge20" class="edge"><title>1-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M94,40C89,45 84,51 81,58 77,67 81,76 87,84"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="90,82 94,92 85,87 90,82"/>
<text text-anchor="middle" x="104" y="71"> query </text>
</g>
<g id="edge14" class="edge"><title>10-&gt;1</title>
<path style="fill:none;stroke:#377d18;" d="M123,92C123,86 124,80 125,74 125,66 125,58 124,50"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="121,50 123,40 127,50 121,50"/>
<text text-anchor="middle" x="146" y="71"> results </text>
</g>
<g id="node6" class="node"><title>5</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="303,180 155,180 155,216 303,216 303,180"/>
<text text-anchor="middle" x="229" y="203" style="font-family:Helvetica;">Realtime.working_set</text>
</g>
<g id="edge16" class="edge"><title>10-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M181,128C188,132 195,138 204,146 211,153 215,162 219,171"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="222,169 223,180 216,172 222,169"/>
<text text-anchor="middle" x="232" y="159"> query </text>
</g>
<g id="node7" class="node"><title>6</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="172,448 22,448 22,484 172,484 172,448"/>
<text text-anchor="middle" x="97" y="471" style="font-family:Helvetica;">Compute.working_set</text>
</g>
<g id="edge18" class="edge"><title>10-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M78,128C46,144 23,157 23,198 23,198 23,198 23,378 23,404 41,426 60,442"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="62,439 68,448 58,445 62,439"/>
<text text-anchor="middle" x="43" y="293"> query </text>
</g>
<g id="node3" class="node"><title>2</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="359,92 233,92 233,128 359,128 359,92"/>
<text text-anchor="middle" x="296" y="115" style="font-family:Helvetica;">realtime_data_src</text>
</g>
<g id="edge2" class="edge"><title>2-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M286,128C280,138 272,151 264,162 261,166 258,169 255,173"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="258,175 248,180 253,170 258,175"/>
<text text-anchor="middle" x="326" y="159">Realtime.Firehose</text>
</g>
<g id="node4" class="node"><title>3</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="280,360 136,360 136,396 280,396 280,360"/>
<text text-anchor="middle" x="208" y="383" style="font-family:Helvetica;">Compute.disk_cache</text>
</g>
<g id="edge8" class="edge"><title>3-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M185,396C169,409 146,427 128,442"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="130,445 120,448 126,439 130,445"/>
<text text-anchor="middle" x="200" y="427">Compute.map</text>
</g>
<g id="node5" class="node"><title>4</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="315,268 159,268 159,308 315,308 315,268"/>
<text text-anchor="middle" x="237" y="285" style="font-family:Helvetica;">indexed segments</text>
<text text-anchor="middle" x="237" y="301" style="font-family:Helvetica;">blob_store (S3, HDFS)</text>
</g>
<g id="edge6" class="edge"><title>4-&gt;3</title>
<path style="fill:none;stroke:#377d18;" d="M230,308C226,321 221,337 217,350"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="220,351 214,360 214,349 220,351"/>
<text text-anchor="middle" x="263" y="339">Compute.load</text>
</g>
<g id="edge10" class="edge"><title>5-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M183,180C174,175 165,169 157,162 149,155 141,146 135,137"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="132,138 130,128 138,135 132,138"/>
<text text-anchor="middle" x="178" y="159"> results </text>
</g>
<g id="edge4" class="edge"><title>5-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M231,216C232,228 233,244 234,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="237,258 235,268 231,258 237,258"/>
<text text-anchor="middle" x="302" y="247">Realtime.segmentPusher</text>
</g>
<g id="edge12" class="edge"><title>6-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M97,448C97,430 98,403 97,378 97,198 97,198 97,198 98,177 104,155 110,138"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="107,137 113,128 113,139 107,137"/>
<text text-anchor="middle" x="119" y="293"> results </text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB

6
examples/rand/src/main/java/druid/examples/RandomFirehoseFactory.java
View File

@ -162,8 +162,8 @@ public class RandomFirehoseFactory implements FirehoseFactory
private final Runnable commitRunnable = new Runnable() { public void run() {} };
private final java.util.Random rand = (seed == 0L) ? new Random() : new Random(seed);
private final LinkedList<String> dimensions = new LinkedList<String>();
private final boolean placeholderForAdd = dimensions.add("inColumn".toLowerCase());
private final boolean placeholderForAdd2 = dimensions.add("target".toLowerCase());
private final boolean placeholderForAdd = dimensions.add("inColumn");
private final boolean placeholderForAdd2 = dimensions.add("target");
private final Map<String, Object> theMap = new HashMap<String, Object>(2);
private long rowCount = 0L;
@ -200,7 +200,7 @@ public class RandomFirehoseFactory implements FirehoseFactory
}
}
rowCount++;
theMap.put("inColumn".toLowerCase(), anotherRand((int)nth));
theMap.put("inColumn", anotherRand((int)nth));
theMap.put("target", ("a" + nth));
return new MapBasedInputRow(System.currentTimeMillis(), dimensions, theMap);
}

15
index-common/pom.xml
View File

@ -79,4 +79,19 @@
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

5
index-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java
View File

@ -59,6 +59,11 @@ public class CompressedLongsIndexedSupplier implements Supplier<IndexedLongs>
this.baseLongBuffers = baseLongBuffers;
}
public int size()
{
return totalSize;
}
@Override
public IndexedLongs get()
{

60
index-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java
View File

@ -22,6 +22,7 @@ package com.metamx.druid.index.v1;
import com.google.common.base.Function;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@ -52,6 +53,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
@ -65,18 +67,19 @@ public class IncrementalIndex implements Iterable<Row>
private final long minTimestamp;
private final QueryGranularity gran;
final AggregatorFactory[] metrics;
private final AggregatorFactory[] metrics;
private final Map<String, Integer> metricIndexes;
private final Map<String, String> metricTypes;
final LinkedHashMap<String, Integer> dimensionOrder;
final CopyOnWriteArrayList<String> dimensions;
final DimensionHolder dimValues;
final ConcurrentSkipListMap<TimeAndDims, Aggregator[]> facts;
private final ImmutableList<String> metricNames;
private final LinkedHashMap<String, Integer> dimensionOrder;
private final CopyOnWriteArrayList<String> dimensions;
private final DimensionHolder dimValues;
private final ConcurrentSkipListMap<TimeAndDims, Aggregator[]> facts;
private volatile int numEntries = 0;
// This is modified by the same thread.
// This is modified on add() by a (hopefully) single thread.
private InputRow in;
public IncrementalIndex(
@ -89,12 +92,16 @@ public class IncrementalIndex implements Iterable<Row>
this.gran = gran;
this.metrics = metrics;
final ImmutableList.Builder<String> metricNamesBuilder = ImmutableList.builder();
final ImmutableMap.Builder<String, Integer> metricIndexesBuilder = ImmutableMap.builder();
final ImmutableMap.Builder<String, String> metricTypesBuilder = ImmutableMap.builder();
for (int i = 0; i < metrics.length; i++) {
metricIndexesBuilder.put(metrics[i].getName().toLowerCase(), i);
metricTypesBuilder.put(metrics[i].getName().toLowerCase(), metrics[i].getTypeName());
final String metricName = metrics[i].getName().toLowerCase();
metricNamesBuilder.add(metricName);
metricIndexesBuilder.put(metricName, i);
metricTypesBuilder.put(metricName, metrics[i].getTypeName());
}
metricNames = metricNamesBuilder.build();
metricIndexes = metricIndexesBuilder.build();
metricTypes = metricTypesBuilder.build();
@ -109,7 +116,7 @@ public class IncrementalIndex implements Iterable<Row>
* Adds a new row. The row might correspond with another row that already exists, in which case this will
* update that row instead of inserting a new one.
* <p/>
* This is *not* thread-safe. Calls to add() must be serialized externally
* This is *not* thread-safe. Calls to add() should always happen on the same thread.
*
* @param row the row of data to add
*
@ -126,6 +133,8 @@ public class IncrementalIndex implements Iterable<Row>
List<String[]> overflow = null;
for (String dimension : rowDimensions) {
dimension = dimension.toLowerCase();
final Integer index = dimensionOrder.get(dimension);
if (index == null) {
dimensionOrder.put(dimension, dimensionOrder.size());
@ -134,9 +143,9 @@ public class IncrementalIndex implements Iterable<Row>
if (overflow == null) {
overflow = Lists.newArrayList();
}
overflow.add(getDimVals(row, dimension));
overflow.add(getDimVals(dimValues.add(dimension), row.getDimension(dimension)));
} else {
dims[index] = getDimVals(row, dimension);
dims[index] = getDimVals(dimValues.get(dimension), row.getDimension(dimension));
}
}
@ -163,8 +172,9 @@ public class IncrementalIndex implements Iterable<Row>
new MetricSelectorFactory()
{
@Override
public FloatMetricSelector makeFloatMetricSelector(final String metricName)
public FloatMetricSelector makeFloatMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
return new FloatMetricSelector()
{
@Override
@ -176,7 +186,7 @@ public class IncrementalIndex implements Iterable<Row>
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(final String metricName)
public ComplexMetricSelector makeComplexMetricSelector(final String metric)
{
final String typeName = agg.getTypeName();
final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);
@ -186,6 +196,7 @@ public class IncrementalIndex implements Iterable<Row>
}
final ComplexMetricExtractor extractor = serde.getExtractor();
final String metricName = metric.toLowerCase();
return new ComplexMetricSelector()
{
@ -227,20 +238,18 @@ public class IncrementalIndex implements Iterable<Row>
return numEntries;
}
private long getMinTimeMillis()
public long getMinTimeMillis()
{
return facts.firstKey().getTimestamp();
}
private long getMaxTimeMillis()
public long getMaxTimeMillis()
{
return facts.lastKey().getTimestamp();
}
private String[] getDimVals(InputRow row, String dimension)
private String[] getDimVals(final DimDim dimLookup, final List<String> dimValues)
{
final DimDim dimLookup = dimValues.getOrAdd(dimension);
final List<String> dimValues = row.getDimension(dimension);
final String[] retVal = new String[dimValues.size()];
int count = 0;
@ -309,11 +318,21 @@ public class IncrementalIndex implements Iterable<Row>
return dimensionOrder.get(dimension);
}
List<String> getMetricNames()
{
return metricNames;
}
Integer getMetricIndex(String metricName)
{
return metricIndexes.get(metricName);
}
ConcurrentSkipListMap<TimeAndDims, Aggregator[]> getFacts()
{
return facts;
}
ConcurrentNavigableMap<TimeAndDims, Aggregator[]> getSubMap(TimeAndDims start, TimeAndDims end)
{
return facts.subMap(start, end);
@ -366,13 +385,16 @@ public class IncrementalIndex implements Iterable<Row>
dimensions.clear();
}
DimDim getOrAdd(String dimension)
DimDim add(String dimension)
{
DimDim holder = dimensions.get(dimension);
if (holder == null) {
holder = new DimDim();
dimensions.put(dimension, holder);
}
else {
throw new ISE("dimension[%s] already existed even though add() was called!?", dimension);
}
return holder;
}

41
index-common/src/main/java/com/metamx/druid/indexer/data/CSVDataSpec.java
View File

@ -19,17 +19,12 @@
package com.metamx.druid.indexer.data;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.metamx.common.parsers.CSVParser;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import org.codehaus.jackson.annotate.JsonCreator;
import org.codehaus.jackson.annotate.JsonProperty;
import javax.annotation.Nullable;
import java.util.List;
/**
@ -45,33 +40,13 @@ public class CSVDataSpec implements DataSpec
@JsonProperty("dimensions") List<String> dimensions
)
{
Preconditions.checkNotNull(columns);
Preconditions.checkArgument(
!Joiner.on("_").join(columns).contains(","), "Columns must not have commas in them"
);
Preconditions.checkNotNull(columns, "columns");
for (String column : columns) {
Preconditions.checkArgument(!column.contains(","), "Column[%s] has a comma, it cannot", column);
}
this.columns = Lists.transform(
columns,
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
this.dimensions = (dimensions == null) ? dimensions : Lists.transform(
dimensions,
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
this.columns = columns;
this.dimensions = dimensions;
}
@JsonProperty("columns")
@ -102,8 +77,8 @@ public class CSVDataSpec implements DataSpec
}
@Override
public Parser getParser()
public Parser<String, Object> getParser()
{
return new ToLowerCaseParser(new CSVParser(columns));
return new CSVParser(columns);
}
}

41
index-common/src/main/java/com/metamx/druid/indexer/data/DelimitedDataSpec.java
View File

@ -19,17 +19,12 @@
package com.metamx.druid.indexer.data;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.metamx.common.parsers.DelimitedParser;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import org.codehaus.jackson.annotate.JsonCreator;
import org.codehaus.jackson.annotate.JsonProperty;
import javax.annotation.Nullable;
import java.util.List;
/**
@ -48,33 +43,13 @@ public class DelimitedDataSpec implements DataSpec
)
{
Preconditions.checkNotNull(columns);
Preconditions.checkArgument(
!Joiner.on("_").join(columns).contains(","), "Columns must not have commas in them"
);
for (String column : columns) {
Preconditions.checkArgument(!column.contains(","), "Column[%s] has a comma, it cannot", column);
}
this.delimiter = (delimiter == null) ? DelimitedParser.DEFAULT_DELIMITER : delimiter;
this.columns = Lists.transform(
columns,
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
this.dimensions = (dimensions == null) ? dimensions : Lists.transform(
dimensions,
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
this.columns = columns;
this.dimensions = dimensions;
}
@JsonProperty("delimiter")
@ -111,10 +86,10 @@ public class DelimitedDataSpec implements DataSpec
}
@Override
public Parser getParser()
public Parser<String, Object> getParser()
{
Parser retVal = new DelimitedParser(delimiter);
Parser<String, Object> retVal = new DelimitedParser(delimiter);
retVal.setFieldNames(columns);
return new ToLowerCaseParser(retVal);
return retVal;
}
}

20
index-common/src/main/java/com/metamx/druid/indexer/data/JSONDataSpec.java
View File

@ -19,14 +19,10 @@
package com.metamx.druid.indexer.data;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.metamx.common.parsers.JSONParser;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import org.codehaus.jackson.annotate.JsonProperty;
import javax.annotation.Nullable;
import java.util.List;
/**
@ -39,17 +35,7 @@ public class JSONDataSpec implements DataSpec
@JsonProperty("dimensions") List<String> dimensions
)
{
this.dimensions = (dimensions == null) ? dimensions : Lists.transform(
dimensions,
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
this.dimensions = dimensions;
}
@JsonProperty("dimensions")
@ -71,8 +57,8 @@ public class JSONDataSpec implements DataSpec
}
@Override
public Parser getParser()
public Parser<String, Object> getParser()
{
return new ToLowerCaseParser(new JSONParser());
return new JSONParser();
}
}

3
index-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java
View File

@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.metamx.common.exception.FormattedException;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import com.metamx.druid.input.InputRow;
import com.metamx.druid.input.MapBasedInputRow;
import org.codehaus.jackson.annotate.JsonCreator;
@ -59,7 +60,7 @@ public class StringInputRowParser
}
this.dimensionExclusions.add(timestampSpec.getTimestampColumn());
this.parser = dataSpec.getParser();
this.parser = new ToLowerCaseParser(dataSpec.getParser());
}
public StringInputRowParser addDimensionExclusion(String dimension)

70
index-common/src/main/java/com/metamx/druid/indexer/data/ToLowercaseDataSpec.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package com.metamx.druid.indexer.data;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import org.codehaus.jackson.annotate.JsonValue;
import java.util.List;
/**
*/
public class ToLowercaseDataSpec implements DataSpec
{
private final DataSpec delegate;
public ToLowercaseDataSpec(
DataSpec delegate
)
{
this.delegate = delegate;
}
@Override
public void verify(List<String> usedCols)
{
delegate.verify(usedCols);
}
@Override
public boolean hasCustomDimensions()
{
return delegate.hasCustomDimensions();
}
@Override
public List<String> getDimensions()
{
return delegate.getDimensions();
}
@Override
public Parser<String, Object> getParser()
{
return new ToLowerCaseParser(delegate.getParser());
}
@JsonValue
public DataSpec getDelegate()
{
return delegate;
}
}

81
index-common/src/test/java/com/metamx/druid/index/v1/IncrementalIndexTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package com.metamx.druid.index.v1;
import com.google.common.collect.ImmutableMap;
import com.metamx.druid.QueryGranularity;
import com.metamx.druid.aggregation.AggregatorFactory;
import com.metamx.druid.input.MapBasedInputRow;
import com.metamx.druid.input.Row;
import junit.framework.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.Iterator;
/**
*/
public class IncrementalIndexTest
{
@Test
public void testCaseInsensitivity() throws Exception
{
final long timestamp = System.currentTimeMillis();
IncrementalIndex index = createCaseInsensitiveIndex(timestamp);
Assert.assertEquals(Arrays.asList("dim1", "dim2"), index.getDimensions());
Assert.assertEquals(2, index.size());
final Iterator<Row> rows = index.iterator();
Row row = rows.next();
Assert.assertEquals(timestamp, row.getTimestampFromEpoch());
Assert.assertEquals(Arrays.asList("1"), row.getDimension("dim1"));
Assert.assertEquals(Arrays.asList("2"), row.getDimension("dim2"));
row = rows.next();
Assert.assertEquals(timestamp, row.getTimestampFromEpoch());
Assert.assertEquals(Arrays.asList("3"), row.getDimension("dim1"));
Assert.assertEquals(Arrays.asList("4"), row.getDimension("dim2"));
}
public static IncrementalIndex createCaseInsensitiveIndex(long timestamp)
{
IncrementalIndex index = new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[]{});
index.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("Dim1", "DiM2"),
ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "DIM1", "3", "dIM2", "4")
)
);
index.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("diM1", "dIM2"),
ImmutableMap.<String, Object>of("Dim1", "1", "DiM2", "2", "dim1", "3", "dim2", "4")
)
);
return index;
}
}

0
server/src/test/resources/druid.sample.tsv → index-common/src/test/resources/druid.sample.tsv
View File

0
server/src/test/resources/druid.sample.tsv.bottom → index-common/src/test/resources/druid.sample.tsv.bottom
View File

0
server/src/test/resources/druid.sample.tsv.top → index-common/src/test/resources/druid.sample.tsv.top
View File

7
indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerConfig.java
View File

@ -36,6 +36,7 @@ import com.metamx.common.logger.Logger;
import com.metamx.druid.RegisteringNode;
import com.metamx.druid.index.v1.serde.Registererer;
import com.metamx.druid.indexer.data.DataSpec;
import com.metamx.druid.indexer.data.ToLowercaseDataSpec;
import com.metamx.druid.indexer.granularity.GranularitySpec;
import com.metamx.druid.indexer.granularity.UniformGranularitySpec;
import com.metamx.druid.indexer.path.PathSpec;
@ -219,7 +220,7 @@ public class HadoopDruidIndexerConfig
public void setTimestampColumnName(String timestampColumnName)
{
this.timestampColumnName = timestampColumnName.toLowerCase();
this.timestampColumnName = timestampColumnName;
}
@JsonProperty()
@ -241,7 +242,7 @@ public class HadoopDruidIndexerConfig
public void setDataSpec(DataSpec dataSpec)
{
this.dataSpec = dataSpec;
this.dataSpec = new ToLowercaseDataSpec(dataSpec);
}
@Deprecated
@ -326,7 +327,7 @@ public class HadoopDruidIndexerConfig
public void setPartitionDimension(String partitionDimension)
{
this.partitionDimension = (partitionDimension == null) ? partitionDimension : partitionDimension.toLowerCase();
this.partitionDimension = (partitionDimension == null) ? partitionDimension : partitionDimension;
}
public boolean partitionByDimension()

7
merger/src/main/java/com/metamx/druid/merger/common/task/DeleteTask.java
View File

@ -76,12 +76,7 @@ public class DeleteTask extends AbstractTask
{
// Strategy: Create an empty segment covering the interval to be deleted
final IncrementalIndex empty = new IncrementalIndex(0, QueryGranularity.NONE, new AggregatorFactory[0]);
final IndexableAdapter emptyAdapter = new IncrementalIndexAdapter(
this.getInterval(),
empty,
new ArrayList<String>(),
new ArrayList<String>()
);
final IndexableAdapter emptyAdapter = new IncrementalIndexAdapter(this.getInterval(), empty);
// Create DataSegment
final DataSegment segment =

8
merger/src/main/java/com/metamx/druid/merger/coordinator/exec/TaskConsumer.java
View File

@ -106,7 +106,7 @@ public class TaskConsumer implements Runnable
catch (Exception e) {
log.makeAlert(e, "Failed to hand off task")
.addData("task", task.getId())
.addData("type", task.getType().toString().toLowerCase())
.addData("type", task.getType().toString())
.addData("dataSource", task.getDataSource())
.addData("interval", task.getInterval())
.emit();
@ -136,7 +136,7 @@ public class TaskConsumer implements Runnable
);
final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder()
.setUser2(task.getDataSource())
.setUser4(task.getType().toString().toLowerCase())
.setUser4(task.getType().toString())
.setUser5(task.getInterval().toString());
// Run preflight checks
@ -233,7 +233,7 @@ public class TaskConsumer implements Runnable
bytes += segment.getSize();
}
builder.setUser3(status.getStatusCode().toString().toLowerCase());
builder.setUser3(status.getStatusCode().toString());
emitter.emit(builder.build("indexer/time/run/millis", status.getDuration()));
emitter.emit(builder.build("indexer/segment/count", status.getSegments().size()));
@ -245,7 +245,7 @@ public class TaskConsumer implements Runnable
String.format("Failed to index: %s", task.getDataSource()),
ImmutableMap.<String, Object>builder()
.put("task", task.getId())
.put("type", task.getType().toString().toLowerCase())
.put("type", task.getType().toString())
.put("dataSource", task.getDataSource())
.put("interval", task.getInterval())
.build()

6
pom.xml
View File

@ -41,11 +41,11 @@
</properties>
<modules>
<module>server</module>
<module>client</module>
<module>common</module>
<module>indexer</module>
<module>index-common</module>
<module>client</module>
<module>indexer</module>
<module>server</module>
<module>merger</module>
<module>realtime</module>
<module>examples</module>

7
server/pom.xml
View File

@ -216,6 +216,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.metamx.druid</groupId>
<artifactId>druid-index-common</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>

2
server/src/main/java/com/metamx/druid/index/brita/AndFilter.java
View File

@ -38,7 +38,7 @@ public class AndFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(BitmapIndexSelector selector)
{
if (filters.size() == 1) {
return filters.get(0).goConcise(selector);

5
server/src/main/java/com/metamx/druid/index/brita/InvertedIndexSelector.java → server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java
View File

@ -19,17 +19,14 @@
package com.metamx.druid.index.brita;
import com.metamx.druid.index.v1.processing.Offset;
import com.metamx.druid.kv.Indexed;
import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
/**
*/
public interface InvertedIndexSelector
public interface BitmapIndexSelector
{
public Indexed<String> getDimensionValues(String dimension);
public int getNumRows();
public int[] getInvertedIndex(String dimension, String value);
public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value);
public Offset getInvertedIndexOffset(String dimension, String value);
}

2
server/src/main/java/com/metamx/druid/index/brita/DimensionPredicateFilter.java
View File

@ -43,7 +43,7 @@ class DimensionPredicateFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(final InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(final BitmapIndexSelector selector)
{
return ImmutableConciseSet.union(
FunctionalIterable.create(selector.getDimensionValues(dimension))

4
server/src/main/java/com/metamx/druid/index/brita/ExtractionFilter.java
View File

@ -47,7 +47,7 @@ public class ExtractionFilter implements Filter
this.fn = fn;
}
private List<Filter> makeFilters(InvertedIndexSelector selector)
private List<Filter> makeFilters(BitmapIndexSelector selector)
{
final Indexed<String> allDimVals = selector.getDimensionValues(dimension);
final List<Filter> filters = Lists.newArrayList();
@ -63,7 +63,7 @@ public class ExtractionFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(BitmapIndexSelector selector)
{
return new OrFilter(makeFilters(selector)).goConcise(selector);
}

2
server/src/main/java/com/metamx/druid/index/brita/Filter.java
View File

@ -25,6 +25,6 @@ import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
*/
public interface Filter
{
public ImmutableConciseSet goConcise(InvertedIndexSelector selector);
public ImmutableConciseSet goConcise(BitmapIndexSelector selector);
public ValueMatcher makeMatcher(ValueMatcherFactory factory);
}

2
server/src/main/java/com/metamx/druid/index/brita/NotFilter.java
View File

@ -35,7 +35,7 @@ public class NotFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(BitmapIndexSelector selector)
{
return ImmutableConciseSet.complement(
baseFilter.goConcise(selector),

2
server/src/main/java/com/metamx/druid/index/brita/OrFilter.java
View File

@ -42,7 +42,7 @@ public class OrFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(BitmapIndexSelector selector)
{
if (filters.size() == 1) {
return filters.get(0).goConcise(selector);

2
server/src/main/java/com/metamx/druid/index/brita/SelectorFilter.java
View File

@ -39,7 +39,7 @@ public class SelectorFilter implements Filter
}
@Override
public ImmutableConciseSet goConcise(InvertedIndexSelector selector)
public ImmutableConciseSet goConcise(BitmapIndexSelector selector)
{
return selector.getConciseInvertedIndex(dimension, value);
}

36
server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexAdapter.java
View File

@ -24,6 +24,7 @@ import com.google.common.collect.Maps;
import com.metamx.common.guava.FunctionalIterable;
import com.metamx.common.logger.Logger;
import com.metamx.druid.aggregation.Aggregator;
import com.metamx.druid.input.Row;
import com.metamx.druid.kv.EmptyIndexedInts;
import com.metamx.druid.kv.Indexed;
import com.metamx.druid.kv.IndexedInts;
@ -46,39 +47,36 @@ public class IncrementalIndexAdapter implements IndexableAdapter
private final Interval dataInterval;
private final IncrementalIndex index;
private final List<String> dimensions;
private final List<String> metrics;
private final Map<String, Map<String, ConciseSet>> invertedIndexes;
public IncrementalIndexAdapter(
Interval dataInterval, IncrementalIndex index, List<String> dimensions, List<String> metrics
Interval dataInterval, IncrementalIndex index
)
{
this.dataInterval = dataInterval;
this.index = index;
this.dimensions = dimensions;
this.metrics = metrics;
this.invertedIndexes = Maps.newHashMap();
for (String dimension : dimensions) {
for (String dimension : index.getDimensions()) {
invertedIndexes.put(dimension, Maps.<String, ConciseSet>newHashMap());
}
int rowNum = 0;
for (IncrementalIndex.TimeAndDims timeAndDims : index.facts.keySet()) {
for (Row row : index) {
}
for (IncrementalIndex.TimeAndDims timeAndDims : index.getFacts().keySet()) {
final String[][] dims = timeAndDims.getDims();
for (String dimension : dimensions) {
if (index.dimensionOrder == null || invertedIndexes == null) {
log.error("wtf, dimensionOrder and indvertedIndexes are null");
}
int dimIndex = index.dimensionOrder.get(dimension);
for (String dimension : index.getDimensions()) {
int dimIndex = index.getDimensionIndex(dimension);
Map<String, ConciseSet> conciseSets = invertedIndexes.get(dimension);
if (conciseSets == null || dims == null) {
log.error("conciseSets and dims are null!");
continue;
}
if (dimIndex >= dims.length || dims[dimIndex] == null) {
continue;
@ -120,19 +118,19 @@ public class IncrementalIndexAdapter implements IndexableAdapter
@Override
public Indexed<String> getAvailableDimensions()
{
return new ListIndexed<String>(dimensions, String.class);
return new ListIndexed<String>(index.getDimensions(), String.class);
}
@Override
public Indexed<String> getAvailableMetrics()
{
return new ListIndexed<String>(metrics, String.class);
return new ListIndexed<String>(index.getMetricNames(), String.class);
}
@Override
public Indexed<String> getDimValueLookup(String dimension)
{
final IncrementalIndex.DimDim dimDim = index.dimValues.get(dimension);
final IncrementalIndex.DimDim dimDim = index.getDimension(dimension);
dimDim.sort();
return new Indexed<String>()
@ -173,7 +171,7 @@ public class IncrementalIndexAdapter implements IndexableAdapter
public Iterable<Rowboat> getRows()
{
return FunctionalIterable
.create(index.facts.entrySet())
.create(index.getFacts().entrySet())
.transform(
new Function<Map.Entry<IncrementalIndex.TimeAndDims, Aggregator[]>, Rowboat>()
{
@ -189,9 +187,9 @@ public class IncrementalIndexAdapter implements IndexableAdapter
final Aggregator[] aggs = input.getValue();
int[][] dims = new int[dimValues.length][];
for (String dimension : dimensions) {
int dimIndex = index.dimensionOrder.get(dimension);
final IncrementalIndex.DimDim dimDim = index.dimValues.get(dimension);
for (String dimension : index.getDimensions()) {
int dimIndex = index.getDimensionIndex(dimension);
final IncrementalIndex.DimDim dimDim = index.getDimension(dimension);
dimDim.sort();
if (dimIndex >= dimValues.length || dimValues[dimIndex] == null) {

19
server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java
View File

@ -85,7 +85,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
@Override
public int getDimensionCardinality(String dimension)
{
IncrementalIndex.DimDim dimDim = index.getDimension(dimension);
IncrementalIndex.DimDim dimDim = index.getDimension(dimension.toLowerCase());
if (dimDim == null) {
return 0;
}
@ -232,8 +232,9 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
}
@Override
public DimensionSelector makeDimensionSelector(String dimensionName)
public DimensionSelector makeDimensionSelector(String dimension)
{
final String dimensionName = dimension.toLowerCase();
final IncrementalIndex.DimDim dimValLookup = index.getDimension(dimensionName);
if (dimValLookup == null) {
return null;
@ -303,8 +304,9 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName)
public FloatMetricSelector makeFloatMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
final Integer metricIndexInt = index.getMetricIndex(metricName);
if (metricIndexInt == null) {
return new FloatMetricSelector()
@ -330,8 +332,9 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName)
public ComplexMetricSelector makeComplexMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
final Integer metricIndexInt = index.getMetricIndex(metricName);
if (metricIndexInt == null) {
return null;
@ -385,7 +388,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
String[] tmpDimensionNames = new String[dimensions.size()];
int i = 0;
for (String dimension : dimensions) {
Integer dimIndex = index.getDimensionIndex(dimension);
Integer dimIndex = index.getDimensionIndex(dimension.toLowerCase());
if (dimIndex != null) {
tmpDimensionNames[i] = dimension;
tmpDimensionIndexes[i] = dimIndex;
@ -495,11 +498,11 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
@Override
public ValueMatcher makeValueMatcher(String dimension, String value)
{
Integer dimIndexObject = index.getDimensionIndex(dimension);
Integer dimIndexObject = index.getDimensionIndex(dimension.toLowerCase());
if (dimIndexObject == null) {
return new BooleanValueMatcher(false);
}
String idObject = index.getDimension(dimension).get(value);
String idObject = index.getDimension(dimension.toLowerCase()).get(value);
if (idObject == null) {
return new BooleanValueMatcher(false);
}
@ -530,7 +533,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
@Override
public ValueMatcher makeValueMatcher(String dimension, final Predicate<String> predicate)
{
Integer dimIndexObject = index.getDimensionIndex(dimension);
Integer dimIndexObject = index.getDimensionIndex(dimension.toLowerCase());
if (dimIndexObject == null) {
return new BooleanValueMatcher(false);
}

24
server/src/main/java/com/metamx/druid/index/v1/IndexMerger.java
View File

@ -112,8 +112,8 @@ public class IndexMerger
final IncrementalIndex index, final Interval dataInterval, File outDir, ProgressIndicator progress
) throws IOException
{
final long firstTimestamp = index.facts.firstKey().getTimestamp();
final long lastTimestamp = index.facts.lastKey().getTimestamp();
final long firstTimestamp = index.getMinTime().getMillis();
final long lastTimestamp = index.getMaxTime().getMillis();
if (!(dataInterval.contains(firstTimestamp) && dataInterval.contains(lastTimestamp))) {
throw new IAE(
"interval[%s] does not encapsulate the full range of timestamps[%s, %s]",
@ -130,26 +130,10 @@ public class IndexMerger
throw new ISE("Can only persist to directories, [%s] wasn't a directory", outDir);
}
final List<String> dimensions = Lists.transform(
Lists.newArrayList(index.dimensionOrder.keySet()),
new Function<String, String>()
{
@Override
public String apply(@Nullable String input)
{
return input.toLowerCase();
}
}
);
final List<String> metrics = Lists.newArrayListWithCapacity(index.metrics.length);
for (int i = 0; i < index.metrics.length; ++i) {
metrics.add(index.metrics[i].getName().toLowerCase());
}
log.info("Starting persist for interval[%s], rows[%,d]", dataInterval, index.size());
return merge(
Arrays.<IndexableAdapter>asList(new IncrementalIndexAdapter(dataInterval, index, dimensions, metrics)),
index.metrics,
Arrays.<IndexableAdapter>asList(new IncrementalIndexAdapter(dataInterval, index)),
index.getMetricAggs(),
outDir,
progress
);

47
server/src/main/java/com/metamx/druid/index/v1/IndexStorageAdapter.java
View File

@ -31,8 +31,8 @@ import com.metamx.common.logger.Logger;
import com.metamx.druid.BaseStorageAdapter;
import com.metamx.druid.Capabilities;
import com.metamx.druid.QueryGranularity;
import com.metamx.druid.index.brita.BitmapIndexSelector;
import com.metamx.druid.index.brita.Filter;
import com.metamx.druid.index.brita.InvertedIndexSelector;
import com.metamx.druid.index.v1.processing.ArrayBasedOffset;
import com.metamx.druid.index.v1.processing.Cursor;
import com.metamx.druid.index.v1.processing.DimensionSelector;
@ -143,7 +143,7 @@ public class IndexStorageAdapter extends BaseStorageAdapter
baseOffset = new ArrayBasedOffset(ids, intervalStartAndEnd.lhs);
} else {
baseOffset = new StartLimitedOffset(
new ConciseOffset(filter.goConcise(new IndexBasedInvertedIndexSelector(index))),
new ConciseOffset(filter.goConcise(new IndexBasedBitmapIndexSelector(index))),
intervalStartAndEnd.lhs
);
}
@ -199,8 +199,9 @@ public class IndexStorageAdapter extends BaseStorageAdapter
}
@Override
public DimensionSelector makeDimensionSelector(final String dimensionName)
public DimensionSelector makeDimensionSelector(String dimension)
{
final String dimensionName = dimension.toLowerCase();
final String[] nameLookup = index.reverseDimLookup.get(dimensionName);
if (nameLookup == null) {
return null;
@ -242,9 +243,10 @@ public class IndexStorageAdapter extends BaseStorageAdapter
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName)
public FloatMetricSelector makeFloatMetricSelector(String metric)
{
IndexedFloats cachedFloats = (IndexedFloats) metricHolderCache.get(metricName);
String metricName = metric.toLowerCase();
IndexedFloats cachedFloats = (IndexedFloats) metricHolderCache.get(metric);
if (cachedFloats == null) {
MetricHolder holder = index.metricVals.get(metricName);
if (holder == null) {
@ -274,8 +276,9 @@ public class IndexStorageAdapter extends BaseStorageAdapter
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName)
public ComplexMetricSelector makeComplexMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
Indexed cachedComplex = (Indexed) metricHolderCache.get(metricName);
if (cachedComplex == null) {
MetricHolder holder = index.metricVals.get(metricName);
@ -335,23 +338,21 @@ public class IndexStorageAdapter extends BaseStorageAdapter
@Override
public Indexed<String> getDimValueLookup(String dimension)
{
return new ListIndexed<String>(Lists.newArrayList(index.dimIdLookup.get(dimension).keySet()), String.class);
return new ListIndexed<String>(
Lists.newArrayList(index.dimIdLookup.get(dimension.toLowerCase()).keySet()), String.class
);
}
@Override
public ImmutableConciseSet getInvertedIndex(String dimension, String dimVal)
{
return index.getInvertedIndex(dimension, dimVal);
return index.getInvertedIndex(dimension.toLowerCase(), dimVal);
}
@Override
public Offset getFilterOffset(Filter filter)
{
return new ConciseOffset(
filter.goConcise(
new IndexBasedInvertedIndexSelector(index)
)
);
return new ConciseOffset(filter.goConcise(new IndexBasedBitmapIndexSelector(index)));
}
@Override
@ -459,11 +460,11 @@ public class IndexStorageAdapter extends BaseStorageAdapter
}
}
private static class IndexBasedInvertedIndexSelector implements InvertedIndexSelector
private static class IndexBasedBitmapIndexSelector implements BitmapIndexSelector
{
private final Index index;
public IndexBasedInvertedIndexSelector(final Index index)
public IndexBasedBitmapIndexSelector(final Index index)
{
this.index = index;
}
@ -473,7 +474,7 @@ public class IndexStorageAdapter extends BaseStorageAdapter
{
return new Indexed<String>()
{
private final String[] dimVals = index.reverseDimLookup.get(dimension);
private final String[] dimVals = index.reverseDimLookup.get(dimension.toLowerCase());
@Override
public Class<? extends String> getClazz()
@ -513,22 +514,10 @@ public class IndexStorageAdapter extends BaseStorageAdapter
return index.timeOffsets.length;
}
@Override
public int[] getInvertedIndex(String dimension, String value)
{
throw new UnsupportedOperationException();
}
@Override
public Offset getInvertedIndexOffset(String dimension, String value)
{
return new ArrayBasedOffset(getInvertedIndex(dimension, value));
}
@Override
public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value)
{
return index.getInvertedIndex(dimension, value);
return index.getInvertedIndex(dimension.toLowerCase(), value);
}
}
}

50
server/src/main/java/com/metamx/druid/index/v1/MMappedIndexStorageAdapter.java
View File

@ -30,8 +30,8 @@ import com.metamx.common.guava.FunctionalIterator;
import com.metamx.druid.BaseStorageAdapter;
import com.metamx.druid.Capabilities;
import com.metamx.druid.QueryGranularity;
import com.metamx.druid.index.brita.BitmapIndexSelector;
import com.metamx.druid.index.brita.Filter;
import com.metamx.druid.index.brita.InvertedIndexSelector;
import com.metamx.druid.index.v1.processing.Cursor;
import com.metamx.druid.index.v1.processing.DimensionSelector;
import com.metamx.druid.index.v1.processing.Offset;
@ -77,7 +77,7 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
@Override
public int getDimensionCardinality(String dimension)
{
final Indexed<String> dimValueLookup = index.getDimValueLookup(dimension);
final Indexed<String> dimValueLookup = index.getDimValueLookup(dimension.toLowerCase());
if (dimValueLookup == null) {
return 0;
}
@ -127,7 +127,7 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
if (filter == null) {
iterable = new NoFilterCursorIterable(index, actualInterval, gran);
} else {
Offset offset = new ConciseOffset(filter.goConcise(new MMappedInvertedIndexSelector(index)));
Offset offset = new ConciseOffset(filter.goConcise(new MMappedBitmapIndexSelector(index)));
iterable = new CursorIterable(index, actualInterval, gran, offset);
}
@ -144,13 +144,13 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
@Override
public Indexed<String> getDimValueLookup(String dimension)
{
return index.getDimValueLookup(dimension);
return index.getDimValueLookup(dimension.toLowerCase());
}
@Override
public ImmutableConciseSet getInvertedIndex(String dimension, String dimVal)
{
return index.getInvertedIndex(dimension, dimVal);
return index.getInvertedIndex(dimension.toLowerCase(), dimVal);
}
@Override
@ -158,7 +158,7 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
{
return new ConciseOffset(
filter.goConcise(
new MMappedInvertedIndexSelector(index)
new MMappedBitmapIndexSelector(index)
)
);
}
@ -241,8 +241,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public DimensionSelector makeDimensionSelector(final String dimensionName)
public DimensionSelector makeDimensionSelector(String dimension)
{
final String dimensionName = dimension;
final Indexed<? extends IndexedInts> rowVals = index.getDimColumn(dimensionName);
final Indexed<String> dimValueLookup = index.getDimValueLookup(dimensionName);
@ -280,8 +281,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName)
public FloatMetricSelector makeFloatMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
IndexedFloats cachedMetricVals = (IndexedFloats) metricHolderCache.get(metricName);
if (cachedMetricVals == null) {
@ -315,8 +317,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName)
public ComplexMetricSelector makeComplexMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
Indexed cachedMetricVals = (Indexed) metricHolderCache.get(metricName);
if (cachedMetricVals == null) {
@ -490,8 +493,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public DimensionSelector makeDimensionSelector(final String dimensionName)
public DimensionSelector makeDimensionSelector(final String dimension)
{
final String dimensionName = dimension.toLowerCase();
final Indexed<? extends IndexedInts> rowVals = index.getDimColumn(dimensionName);
final Indexed<String> dimValueLookup = index.getDimValueLookup(dimensionName);
@ -529,8 +533,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName)
public FloatMetricSelector makeFloatMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
IndexedFloats cachedMetricVals = (IndexedFloats) metricCacheMap.get(metricName);
if (cachedMetricVals == null) {
@ -566,8 +571,9 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName)
public ComplexMetricSelector makeComplexMetricSelector(String metric)
{
final String metricName = metric.toLowerCase();
Indexed cachedMetricVals = (Indexed) metricCacheMap.get(metricName);
if (cachedMetricVals == null) {
@ -625,11 +631,11 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
}
}
private static class MMappedInvertedIndexSelector implements InvertedIndexSelector
private static class MMappedBitmapIndexSelector implements BitmapIndexSelector
{
private final MMappedIndex index;
public MMappedInvertedIndexSelector(final MMappedIndex index)
public MMappedBitmapIndexSelector(final MMappedIndex index)
{
this.index = index;
}
@ -637,7 +643,7 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
@Override
public Indexed<String> getDimensionValues(String dimension)
{
return index.getDimValueLookup(dimension);
return index.getDimValueLookup(dimension.toLowerCase());
}
@Override
@ -646,22 +652,10 @@ public class MMappedIndexStorageAdapter extends BaseStorageAdapter
return index.getReadOnlyTimestamps().size();
}
@Override
public int[] getInvertedIndex(String dimension, String value)
{
throw new UnsupportedOperationException();
}
@Override
public Offset getInvertedIndexOffset(String dimension, String value)
{
return new ConciseOffset(index.getInvertedIndex(dimension, value));
}
@Override
public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value)
{
return index.getInvertedIndex(dimension, value);
return index.getInvertedIndex(dimension.toLowerCase(), value);
}
}
}

7
server/src/test/java/com/metamx/druid/index/v1/EmptyIndexTest.java
View File

@ -45,12 +45,7 @@ public class EmptyIndexTest
tmpDir.deleteOnExit();
IncrementalIndex emptyIndex = new IncrementalIndex(0, QueryGranularity.NONE, new AggregatorFactory[0]);
IncrementalIndexAdapter emptyIndexAdapter = new IncrementalIndexAdapter(
new Interval("2012-08-01/P3D"),
emptyIndex,
new ArrayList<String>(),
new ArrayList<String>()
);
IncrementalIndexAdapter emptyIndexAdapter = new IncrementalIndexAdapter(new Interval("2012-08-01/P3D"), emptyIndex);
IndexMerger.merge(Lists.<IndexableAdapter>newArrayList(emptyIndexAdapter), new AggregatorFactory[0], tmpDir);
MMappedIndex emptyIndexMMapped = IndexIO.mapDir(tmpDir);

113
server/src/test/java/com/metamx/druid/index/v1/IndexMergerTest.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package com.metamx.druid.index.v1;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import com.metamx.druid.QueryGranularity;
import com.metamx.druid.aggregation.AggregatorFactory;
import com.metamx.druid.input.MapBasedInputRow;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import java.io.File;
import java.util.Arrays;
/**
*/
public class IndexMergerTest
{
@Test
public void testPersistCaseInsensitive() throws Exception
{
final long timestamp = System.currentTimeMillis();
IncrementalIndex toPersist = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);
final File tempDir = Files.createTempDir();
try {
MMappedIndex index = IndexIO.mapDir(IndexMerger.persist(toPersist, tempDir));
Assert.assertEquals(2, index.getTimestamps().size());
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
Assert.assertEquals(0, index.getAvailableMetrics().size());
}
finally {
tempDir.delete();
}
}
@Test
public void testPersistMergeCaseInsensitive() throws Exception
{
final long timestamp = System.currentTimeMillis();
IncrementalIndex toPersist1 = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);
IncrementalIndex toPersist2 = new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[]{});
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("DIm1", "DIM2"),
ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "DIm1", "10000", "DIM2", "100000000")
)
);
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("dIM1", "dIm2"),
ImmutableMap.<String, Object>of("DIm1", "1", "DIM2", "2", "dim1", "5", "dim2", "6")
)
);
final File tempDir1 = Files.createTempDir();
final File tempDir2 = Files.createTempDir();
final File mergedDir = Files.createTempDir();
try {
MMappedIndex index1 = IndexIO.mapDir(IndexMerger.persist(toPersist1, tempDir1));
Assert.assertEquals(2, index1.getTimestamps().size());
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
Assert.assertEquals(0, index1.getAvailableMetrics().size());
MMappedIndex index2 = IndexIO.mapDir(IndexMerger.persist(toPersist2, tempDir2));
Assert.assertEquals(2, index2.getTimestamps().size());
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index2.getAvailableDimensions()));
Assert.assertEquals(0, index2.getAvailableMetrics().size());
MMappedIndex merged = IndexIO.mapDir(
IndexMerger.mergeMMapped(Arrays.asList(index1, index2), new AggregatorFactory[]{}, mergedDir)
);
Assert.assertEquals(3, merged.getTimestamps().size());
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
Assert.assertEquals(0, merged.getAvailableMetrics().size());
}
finally {
FileUtils.deleteQuietly(tempDir1);
FileUtils.deleteQuietly(tempDir2);
FileUtils.deleteQuietly(mergedDir);
}
}
}

71
server/src/test/java/com/metamx/druid/index/v1/TestIndex.java
View File

@ -19,16 +19,21 @@
package com.metamx.druid.index.v1;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
import com.google.common.io.CharStreams;
import com.google.common.io.Closeables;
import com.google.common.io.InputSupplier;
import com.google.common.io.LineProcessor;
import com.google.common.primitives.Ints;
import com.metamx.common.ISE;
import com.metamx.common.logger.Logger;
import com.metamx.common.parsers.DelimitedParser;
import com.metamx.common.parsers.Parser;
import com.metamx.common.parsers.ToLowerCaseParser;
import com.metamx.druid.QueryGranularity;
import com.metamx.druid.aggregation.AggregatorFactory;
import com.metamx.druid.aggregation.DoubleSumAggregatorFactory;
@ -36,7 +41,11 @@ import com.metamx.druid.client.RangeIterable;
import com.metamx.druid.guava.GuavaUtils;
import com.metamx.druid.index.v1.serde.ComplexMetricSerde;
import com.metamx.druid.index.v1.serde.ComplexMetrics;
import com.metamx.druid.indexer.data.DelimitedDataSpec;
import com.metamx.druid.indexer.data.StringInputRowParser;
import com.metamx.druid.indexer.data.TimestampSpec;
import com.metamx.druid.input.InputRow;
import com.metamx.druid.input.MapBasedInputRow;
import com.metamx.druid.kv.ArrayIndexed;
import com.metamx.druid.kv.Indexed;
import com.metamx.druid.kv.IndexedFloats;
@ -50,6 +59,7 @@ import org.joda.time.Interval;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
@ -71,8 +81,9 @@ public class TestIndex
private static MMappedIndex mmappedIndex = null;
private static MMappedIndex mergedRealtime = null;
public static final String[] DIMENSIONS = new String[]{"provider", "quality", "placement", "placementish"};
public static final String[] METRICS = new String[]{"index"};
public static final String[] COLUMNS = new String[]{"ts", "provider", "quALIty", "plAcEmEnT", "pLacementish", "iNdEx"};
public static final String[] DIMENSIONS = new String[]{"provider", "quALIty", "plAcEmEnT", "pLacementish"};
public static final String[] METRICS = new String[]{"iNdEx"};
public static final Map<String, Integer> dimIds = Maps.uniqueIndex(
new RangeIterable(4),
new Function<Integer, String>()
@ -310,7 +321,9 @@ public class TestIndex
Arrays.asList(
com.metamx.druid.index.v1.IndexIO.mapDir(topFile),
com.metamx.druid.index.v1.IndexIO.mapDir(bottomFile)
), METRIC_AGGS, mergedFile
),
METRIC_AGGS,
mergedFile
)
);
@ -324,9 +337,8 @@ public class TestIndex
private static IncrementalIndex makeRealtimeIndex(final String resourceFilename)
{
URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename);
String filename = resource.getFile();
log.info("Realtime loading index file[%s]", filename);
final URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename);
log.info("Realtime loading index file[%s]", resource);
final IncrementalIndex retVal = new IncrementalIndex(
new DateTime("2011-01-12T00:00:00.000Z").getMillis(), QueryGranularity.NONE, METRIC_AGGS
@ -336,9 +348,24 @@ public class TestIndex
int lineCount;
try {
lineCount = CharStreams.readLines(
GuavaUtils.joinFiles(new File(filename)),
CharStreams.newReaderSupplier(
new InputSupplier<InputStream>()
{
@Override
public InputStream getInput() throws IOException
{
return resource.openStream();
}
},
Charsets.UTF_8
),
new LineProcessor<Integer>()
{
StringInputRowParser parser = new StringInputRowParser(
new TimestampSpec("ts", "iso"),
new DelimitedDataSpec("\t", Arrays.asList(COLUMNS), Arrays.asList(DIMENSIONS)),
Arrays.<String>asList()
);
boolean runOnce = false;
int lineCount = 0;
@ -352,35 +379,7 @@ public class TestIndex
final String[] splits = line.split("\t");
retVal.add(
new InputRow()
{
@Override
public long getTimestampFromEpoch()
{
return new DateTime(splits[0]).getMillis();
}
@Override
public List<String> getDimensions()
{
return Arrays.asList(DIMENSIONS);
}
@Override
public List<String> getDimension(String dimension)
{
return Arrays.asList(splits[dimIds.get(dimension) + 1].split("\u0001"));
}
@Override
public float getFloatMetric(String metric)
{
Preconditions.checkArgument(METRICS[0].equals(metric), "WTF!?");
return Float.parseFloat(splits[5]);
}
}
);
retVal.add(parser.parse(line));
++lineCount;
return true;

2
server/src/test/java/com/metamx/druid/query/QueryRunnerTestHelper.java
View File

@ -54,7 +54,7 @@ public class QueryRunnerTestHelper
public static final String dataSource = "testing";
public static final QueryGranularity gran = QueryGranularity.DAY;
public static final QueryGranularity allGran = QueryGranularity.ALL;
public static final String providerDimension = "provider";
public static final String providerDimension = "proVider";
public static final String qualityDimension = "quality";
public static final String placementishDimension = "placementish";
public static final String indexMetric = "index";

45
server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java
View File

@ -21,6 +21,7 @@ package com.metamx.druid.query.search;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.metamx.common.guava.Sequences;
import com.metamx.druid.Druids;
@ -42,6 +43,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
/**
*/
@ -75,12 +77,12 @@ public class SearchQueryRunnerTest
.query("a")
.build();
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(
QueryRunnerTestHelper.qualityDimension,
Sets.newHashSet("automotive", "mezzanine", "travel", "health", "entertainment")
);
expectedResults.put(QueryRunnerTestHelper.providerDimension, Sets.newHashSet("total_market"));
expectedResults.put(QueryRunnerTestHelper.providerDimension.toLowerCase(), Sets.newHashSet("total_market"));
expectedResults.put(QueryRunnerTestHelper.placementishDimension, Sets.newHashSet("a"));
checkSearchQuery(searchQuery, expectedResults);
@ -96,7 +98,7 @@ public class SearchQueryRunnerTest
.query(new FragmentSearchQuerySpec(Arrays.asList("auto", "ve"), null))
.build();
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.qualityDimension, Sets.newHashSet("automotive"));
checkSearchQuery(searchQuery, expectedResults);
@ -129,7 +131,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithDimension2()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet<String>(Arrays.asList("total_market")));
checkSearchQuery(
@ -147,7 +149,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithDimensions1()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.putAll(
ImmutableMap.<String, Set<String>>of(
QueryRunnerTestHelper.qualityDimension,
@ -167,7 +169,12 @@ public class SearchQueryRunnerTest
Druids.newSearchQueryBuilder()
.dataSource(QueryRunnerTestHelper.dataSource)
.granularity(QueryRunnerTestHelper.allGran)
.dimensions(Arrays.asList(QueryRunnerTestHelper.qualityDimension, QueryRunnerTestHelper.providerDimension))
.dimensions(
Arrays.asList(
QueryRunnerTestHelper.qualityDimension,
QueryRunnerTestHelper.providerDimension
)
)
.intervals(QueryRunnerTestHelper.fullOnInterval)
.query("a")
.build(),
@ -178,14 +185,19 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithDimensions2()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet<String>(Arrays.asList("total_market")));
checkSearchQuery(
Druids.newSearchQueryBuilder()
.dataSource(QueryRunnerTestHelper.dataSource)
.granularity(QueryRunnerTestHelper.allGran)
.dimensions(Arrays.asList(QueryRunnerTestHelper.placementishDimension, QueryRunnerTestHelper.providerDimension))
.dimensions(
Arrays.asList(
QueryRunnerTestHelper.placementishDimension,
QueryRunnerTestHelper.providerDimension
)
)
.intervals(QueryRunnerTestHelper.fullOnInterval)
.query("mark")
.build(),
@ -196,7 +208,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithSingleFilter1()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(
QueryRunnerTestHelper.qualityDimension, new HashSet<String>(Arrays.asList("automotive"))
);
@ -217,7 +229,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithSingleFilter2()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet<String>(Arrays.asList("total_market")));
checkSearchQuery(
@ -236,7 +248,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchMultiAndFilter()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.qualityDimension, new HashSet<String>(Arrays.asList("automotive")));
DimFilter filter = Druids.newAndDimFilterBuilder()
@ -270,7 +282,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithMultiOrFilter()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
expectedResults.put(QueryRunnerTestHelper.qualityDimension, new HashSet<String>(Arrays.asList("automotive")));
DimFilter filter = Druids.newOrDimFilterBuilder()
@ -304,7 +316,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithEmptyResults()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
checkSearchQuery(
Druids.newSearchQueryBuilder()
@ -320,7 +332,7 @@ public class SearchQueryRunnerTest
@Test
public void testSearchWithFilterEmptyResults()
{
Map<String, Set<String>> expectedResults = new HashMap<String, Set<String>>();
Map<String, Set<String>> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
DimFilter filter = Druids.newAndDimFilterBuilder()
.fields(
@ -364,7 +376,10 @@ public class SearchQueryRunnerTest
for (SearchHit resultValue : resultValues) {
String dimension = resultValue.getDimension();
String theValue = resultValue.getValue();
Assert.assertTrue(expectedResults.containsKey(dimension));
Assert.assertTrue(
String.format("Result had unknown dimension[%s]", dimension),
expectedResults.containsKey(dimension)
);
Set<String> expectedSet = expectedResults.get(dimension);
Assert.assertTrue(

4
server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerTest.java
View File

@ -61,9 +61,7 @@ public class TimeseriesQueryRunnerTest
@Parameterized.Parameters
public static Collection<?> constructorFeeder() throws IOException
{
return QueryRunnerTestHelper.makeQueryRunners(
new TimeseriesQueryRunnerFactory()
);
return QueryRunnerTestHelper.makeQueryRunners(new TimeseriesQueryRunnerFactory());
}
private final QueryRunner runner;