diff --git a/docs/content/Aggregations.md b/docs/content/Aggregations.md index 29740a2858c..c8c79c6acf0 100644 --- a/docs/content/Aggregations.md +++ b/docs/content/Aggregations.md @@ -159,4 +159,29 @@ Uses [HyperLogLog](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) to ```json { "type" : "hyperUnique", "name" : , "fieldName" : } -``` \ No newline at end of file +``` + +## Miscellaneous Aggregations + +### Filtered Aggregator + +A filtered aggregator wraps any given aggregator, but only aggregates the values for which the given dimension filter matches. + +This makes it possible to compute the results of a filtered and an unfiltered aggregation simultaneously, without having to issue multiple queries, and use both results as part of post-aggregations. + +*Limitations:* The filtered aggregator currently only supports selector and not filter with a single selector, i.e. matching a dimension against a single value. + +*Note:* If only the filtered results are required, consider putting the filter on the query itself, which will be much faster since it does not require scanning all the data. + +```json +{ + "type" : "filtered", + "name" : "aggMatching", + "filter" : { + "type" : "selector", + "dimension" : , + "value" : + } + "aggregator" : +} +``` diff --git a/docs/content/Batch-ingestion.md b/docs/content/Batch-ingestion.md index 2f53eb48b4e..0326d1e1a66 100644 --- a/docs/content/Batch-ingestion.md +++ b/docs/content/Batch-ingestion.md @@ -162,37 +162,58 @@ The indexing process has the ability to roll data up as it processes the incomin ### Partitioning specification -Segments are always partitioned based on timestamp (according to the granularitySpec) and may be further partitioned in some other way depending on partition type. -Druid supports two types of partitions spec - singleDimension and hashed. +Segments are always partitioned based on timestamp (according to the granularitySpec) and may be further partitioned in +some other way depending on partition type. Druid supports two types of partitioning strategies: "hashed" (based on the +hash of all dimensions in each row), and "dimension" (based on ranges of a single dimension). -In SingleDimension partition type data is partitioned based on the values in that dimension. -For example, data for a day may be split by the dimension "last\_name" into two segments: one with all values from A-M and one with all values from N-Z. +Hashed partitioning is recommended in most cases, as it will improve indexing performance and create more uniformly +sized data segments relative to single-dimension partitioning. -In hashed partition type, the number of partitions is determined based on the targetPartitionSize and cardinality of input set and the data is partitioned based on the hashcode of the row. - -It is recommended to use Hashed partition as it is more efficient than singleDimension since it does not need to determine the dimension for creating partitions. -Hashing also gives better distribution of data resulting in equal sized partitions and improving query performance - -To use this druid to automatically determine optimal partitions indexer must be given a target partition size. It can then find a good set of partition ranges on its own. - -#### Configuration for disabling auto-sharding and creating Fixed number of partitions - Druid can be configured to NOT run determine partitions and create a fixed number of shards by specifying numShards in hashed partitionsSpec. - e.g This configuration will skip determining optimal partitions and always create 4 shards for every segment granular interval +#### Hash-based partitioning ```json "partitionsSpec": { - "type": "hashed" - "numShards": 4 + "type": "hashed", + "targetPartitionSize": 5000000 } ``` +Hashed partitioning works by first selecting a number of segments, and then partitioning rows across those segments +according to the hash of all dimensions in each row. The number of segments is determined automatically based on the +cardinality of the input set and a target partition size. + +The configuration options are: + |property|description|required?| |--------|-----------|---------| -|type|type of partitionSpec to be used |no, default : singleDimension| -|targetPartitionSize|target number of rows to include in a partition, should be a number that targets segments of 700MB\~1GB.|yes| +|type|type of partitionSpec to be used |"hashed"| +|targetPartitionSize|target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|either this or numShards| +|numShards|specify the number of partitions directly, instead of a target partition size. Ingestion will run faster, since it can skip the step necessary to select a number of partitions automatically.|either this or targetPartitionSize| + +#### Single-dimension partitioning + +```json + "partitionsSpec": { + "type": "dimension", + "targetPartitionSize": 5000000 + } +``` + +Single-dimension partitioning works by first selecting a dimension to partition on, and then separating that dimension +into contiguous ranges. Each segment will contain all rows with values of that dimension in that range. For example, +your segments may be partitioned on the dimension "host" using the ranges "a.example.com" to "f.example.com" and +"f.example.com" to "z.example.com". By default, the dimension to use is determined automatically, although you can +override it with a specific dimension. + +The configuration options are: + +|property|description|required?| +|--------|-----------|---------| +|type|type of partitionSpec to be used |"dimension"| +|targetPartitionSize|target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|yes| +|maxPartitionSize|maximum number of rows to include in a partition. Defaults to 50% larger than the targetPartitionSize.|no| |partitionDimension|the dimension to partition on. Leave blank to select a dimension automatically.|no| -|assumeGrouped|assume input data has already been grouped on time and dimensions. This is faster, but can choose suboptimal partitions if the assumption is violated.|no| -|numShards|provides a way to manually override druid-auto sharding and specify the number of shards to create for each segment granular interval.It is only supported by hashed partitionSpec and targetPartitionSize must be set to -1|no| +|assumeGrouped|assume input data has already been grouped on time and dimensions. Ingestion will run faster, but can choose suboptimal partitions if the assumption is violated.|no| ### Updater job spec diff --git a/docs/content/Coordinator.md b/docs/content/Coordinator.md index 9021cbe1dff..42d1d17041d 100644 --- a/docs/content/Coordinator.md +++ b/docs/content/Coordinator.md @@ -20,9 +20,7 @@ io.druid.cli.Main server coordinator Rules ----- -Segments are loaded and dropped from the cluster based on a set of rules. Rules indicate how segments should be assigned to different historical node tiers and how many replicants of a segment should exist in each tier. Rules may also indicate when segments should be dropped entirely from the cluster. The coordinator loads a set of rules from the database. Rules may be specific to a certain datasource and/or a default set of rules can be configured. Rules are read in order and hence the ordering of rules is important. The coordinator will cycle through all available segments and match each segment with the first rule that applies. Each segment may only match a single rule. - -For more information on rules, see [Rule Configuration](Rule-Configuration.html). +Segments can be automatically loaded and dropped from the cluster based on a set of rules. For more information on rules, see [Rule Configuration](Rule-Configuration.html). Cleaning Up Segments -------------------- diff --git a/docs/content/Druid-vs-Cassandra.md b/docs/content/Druid-vs-Cassandra.md index 4ef29a1ab36..f8d1d59a473 100644 --- a/docs/content/Druid-vs-Cassandra.md +++ b/docs/content/Druid-vs-Cassandra.md @@ -1,6 +1,11 @@ --- layout: doc_page --- + +Druid vs. Cassandra +=================== + + We are not experts on Cassandra, if anything is incorrect about our portrayal, please let us know on the mailing list or via some other means. We will fix this page. Druid is highly optimized for scans and aggregations, it supports arbitrarily deep drill downs into data sets without the need to pre-compute, and it can ingest event streams in real-time and allow users to query events as they come in. Cassandra is a great key-value store and it has some features that allow you to use it to do more interesting things than what you can do with a pure key-value store. But, it is not built for the same use cases that Druid handles, namely regularly scanning over billions of entries per query. diff --git a/docs/content/Druid-vs-Hadoop.md b/docs/content/Druid-vs-Hadoop.md index 3b4ef1df76c..47d03dd7704 100644 --- a/docs/content/Druid-vs-Hadoop.md +++ b/docs/content/Druid-vs-Hadoop.md @@ -2,6 +2,10 @@ layout: doc_page --- +Druid vs Hadoop +=============== + + Hadoop has shown the world that it’s possible to house your data warehouse on commodity hardware for a fraction of the price of typical solutions. As people adopt Hadoop for their data warehousing needs, they find two things. 1. They can now query all of their data in a fairly flexible manner and answer any question they have diff --git a/docs/content/Druid-vs-Impala-or-Shark.md b/docs/content/Druid-vs-Impala-or-Shark.md index cb658c8e087..ab2cb122524 100644 --- a/docs/content/Druid-vs-Impala-or-Shark.md +++ b/docs/content/Druid-vs-Impala-or-Shark.md @@ -1,6 +1,10 @@ --- layout: doc_page --- + +Druid vs Impala or Shark +======================== + The question of Druid versus Impala or Shark basically comes down to your product requirements and what the systems were designed to do. Druid was designed to diff --git a/docs/content/Druid-vs-Redshift.md b/docs/content/Druid-vs-Redshift.md index 4fe06586467..faaaa3f0f33 100644 --- a/docs/content/Druid-vs-Redshift.md +++ b/docs/content/Druid-vs-Redshift.md @@ -1,6 +1,10 @@ --- layout: doc_page --- +Druid vs Redshift +================= + + ###How does Druid compare to Redshift? In terms of drawing a differentiation, Redshift is essentially ParAccel (Actian) which Amazon is licensing. diff --git a/docs/content/Druid-vs-Vertica.md b/docs/content/Druid-vs-Vertica.md index e6971ae03d9..4fe0eb78892 100644 --- a/docs/content/Druid-vs-Vertica.md +++ b/docs/content/Druid-vs-Vertica.md @@ -1,6 +1,11 @@ --- layout: doc_page --- + +Druid vs Vertica +================ + + How does Druid compare to Vertica? Vertica is similar to ParAccel/Redshift ([Druid-vs-Redshift](Druid-vs-Redshift.html)) described above in that it wasn’t built for real-time streaming data ingestion and it supports full SQL. diff --git a/docs/content/Examples.md b/docs/content/Examples.md index 468bdb8f41a..45c5d573892 100644 --- a/docs/content/Examples.md +++ b/docs/content/Examples.md @@ -19,13 +19,13 @@ Clone Druid and build it: git clone https://github.com/metamx/druid.git druid cd druid git fetch --tags -git checkout druid-0.6.156 +git checkout druid-0.6.160 ./build.sh ``` ### Downloading the DSK (Druid Standalone Kit) -[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.156-bin.tar.gz) a stand-alone tarball and run it: +[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.160-bin.tar.gz) a stand-alone tarball and run it: ``` bash tar -xzf druid-services-0.X.X-bin.tar.gz diff --git a/docs/content/Granularities.md b/docs/content/Granularities.md index d4f3ea73141..e8a3e4fc2bf 100644 --- a/docs/content/Granularities.md +++ b/docs/content/Granularities.md @@ -21,13 +21,13 @@ Duration granularities are specified as an exact duration in milliseconds and ti They also support specifying an optional origin, which defines where to start counting time buckets from (defaults to 1970-01-01T00:00:00Z). -``` +```javascript {"type": "duration", "duration": "7200000"} ``` This chunks up every 2 hours. -``` +```javascript {"type": "duration", "duration": "3600000", "origin": "2012-01-01T00:30:00Z"} ``` @@ -39,13 +39,13 @@ Period granularities are specified as arbitrary period combinations of years, mo Time zone is optional (defaults to UTC). Origin is optional (defaults to 1970-01-01T00:00:00 in the given time zone). -``` +```javascript {"type": "period", "period": "P2D", "timeZone": "America/Los_Angeles"} ``` This will bucket by two-day chunks in the Pacific timezone. -``` +```javascript {"type": "period", "period": "P3M", "timeZone": "America/Los_Angeles", "origin": "2012-02-01T00:00:00-08:00"} ``` diff --git a/docs/content/Hadoop-Configuration.md b/docs/content/Hadoop-Configuration.md new file mode 100644 index 00000000000..ef8c48e167a --- /dev/null +++ b/docs/content/Hadoop-Configuration.md @@ -0,0 +1,360 @@ +--- +layout: doc_page +--- + +Example Production Hadoop Configuration +======================================= + +The following configuration should work relatively well for Druid indexing and Hadoop. In the example, we are using Hadoop 2.4 with EC2 m1.xlarge nodes for NameNodes and cc2.8xlarge nodes for DataNodes. + +### Core-site.xml + +``` + + + + + hadoop.tmp.dir + /mnt/persistent/hadoop + + + + + fs.defaultFS + hdfs://#{IP}:9000 + + + fs.s3.impl + org.apache.hadoop.fs.s3native.NativeS3FileSystem + + + fs.s3.awsAccessKeyId + #{S3_ACCESS_KEY} + + + fs.s3.awsSecretAccessKey + #{S3_SECRET_KEY} + + + fs.s3.buffer.dir + /mnt/persistent/hadoop-s3n + + + fs.s3n.awsAccessKeyId + #{S3N_ACCESS_KEY} + + + fs.s3n.awsSecretAccessKey + #{S3N_SECRET_KEY} + + + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.Lz4Codec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec + + + + + io.seqfile.local.dir + /mnt/persistent/hadoop/io/local + + + +``` + +### Mapred-site.xml + +``` + + + + mapreduce.framework.name + yarn + + + + mapreduce.jobtracker.address + #{JT_ADDR}:9001 + + + mapreduce.jobtracker.http.address + #{JT_HTTP_ADDR}:9100 + + + mapreduce.jobhistory.address + #{JH_ADDR}:10020 + + + mapreduce.jobhistory.webapp.address + #{JH_WEBAPP_ADDR}:19888 + + + mapreduce.tasktracker.http.address + #{TT_ADDR}:9103 + + + + + mapreduce.job.reduces + 21 + + + + mapreduce.job.jvm.numtasks + 20 + + + mapreduce.map.memory.mb + 2048 + + + mapreduce.map.java.opts + -server -Xmx1536m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps + + + mapreduce.reduce.memory.mb + 6144 + + + mapreduce.reduce.java.opts + -server -Xmx2560m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps + + + mapreduce.reduce.shuffle.parallelcopies + 50 + + + mapreduce.reduce.shuffle.input.buffer.percent + 0.5 + + + mapreduce.task.io.sort.mb + 256 + + + mapreduce.task.io.sort.factor + 100 + + + mapreduce.jobtracker.handler.count + 64 + + + mapreduce.tasktracker.http.threads + 20 + + + + + mapreduce.cluster.local.dir + /mnt/persistent/hadoop/mapred/local + + + + + mapreduce.jobhistory.recovery.enable + true + + + mapreduce.jobhistory.recovery.store.class + org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService + + + mapreduce.jobhistory.recovery.store.fs.uri + file://${hadoop.tmp.dir}/mapred-jobhistory-state + + + + + + mapreduce.output.fileoutputformat.compress + false + + + mapreduce.map.output.compress + true + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.Lz4Codec + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.GzipCodec + + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + + + mapreduce.task.timeout + 1800000 + + + +``` + +### Yarn-site.xml + +``` + + + + yarn.resourcemanager.hostname + #{RM_HOSTNAME} + + + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.log-aggregation-enable + true + + + yarn.log.server.url + http://#{IP_LOG_SERVER}:19888/jobhistory/logs/ + + + yarn.nodemanager.hostname + #{IP_ADDR} + + + yarn.scheduler.minimum-allocation-mb + 512 + + + yarn.nodemanager.resource.memory-mb + 1024 + + + yarn.nodemanager.resource.cpu-vcores + 1 + + + yarn.nodemanager.vmem-check-enabled + false + + + + + yarn.nodemanager.local-dirs + /mnt/persistent/hadoop/nm-local-dir + + + + + yarn.resourcemanager.recovery.enabled + false + + + yarn.resourcemanager.store.class + org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore + + + yarn.resourcemanager.fs.state-store.uri + file://${hadoop.tmp.dir}/yarn-resourcemanager-state + + + + + yarn.resourcemanager.nodes.exclude-path + /mnt/persistent/hadoop/yarn-exclude.txt + + + +``` + +### HDFS-site.xml + +``` + + + + dfs.replication + 3 + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + + dfs.hosts.exclude + /mnt/persistent/hadoop/hdfs-exclude.txt + + + + + dfs.datanode.data.dir + file:///mnt/persistent/hadoop/dfs/data + + + +``` + +### Capacity-scheduler.xml + +``` + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.1 + + + yarn.scheduler.capacity.root.queues + default + + + yarn.scheduler.capacity.root.default.capacity + 100 + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + + yarn.scheduler.capacity.root.queues + default + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + + yarn.scheduler.capacity.root.default.state + RUNNING + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + + yarn.scheduler.capacity.root.default.acl_administer_queue + * + + + yarn.scheduler.capacity.node-locality-delay + -1 + + + +``` \ No newline at end of file diff --git a/docs/content/Ingestion-FAQ.md b/docs/content/Ingestion-FAQ.md index fc83907c47a..972e62a6a23 100644 --- a/docs/content/Ingestion-FAQ.md +++ b/docs/content/Ingestion-FAQ.md @@ -1,6 +1,11 @@ --- layout: doc_page --- + +## What types of data does Druid support? + +Druid can ingest JSON, CSV, TSV and other delimited data out of the box. Druid supports single dimension values, or multiple dimension values (an array of strings). Druid supports long and float numeric columns. + ## Where do my Druid segments end up after ingestion? Depending on what `druid.storage.type` is set to, Druid will upload segments to some [Deep Storage](Deep-Storage.html). Local disk is used as the default deep storage. @@ -21,6 +26,14 @@ druid.storage.bucket=druid druid.storage.baseKey=sample ``` +Other common reasons that hand-off fails are as follows: + +1) Historical nodes are out of capacity and cannot download any more segments. You'll see exceptions in the coordinator logs if this occurs. + +2) Segments are corrupt and cannot download. You'll see exceptions in your historical nodes if this occurs. + +3) Deep storage is improperly configured. Make sure that your segment actually exists in deep storage and that the coordinator logs have no errors. + ## How do I get HDFS to work? Make sure to include the `druid-hdfs-storage` module as one of your extensions and set `druid.storage.type=hdfs`. @@ -35,7 +48,7 @@ You can check the coordinator console located at `:/cluste ## My queries are returning empty results -You can check `:/druid/v2/datasources/?interval=0/3000` for the dimensions and metrics that have been created for your datasource. Make sure that the name of the aggregators you use in your query match one of these metrics. Also make sure that the query interval you specify match a valid time range where data exists. Note: the broker endpoint will only return valid results on historical segments. +You can check `:/druid/v2/datasources/?interval=0/3000` for the dimensions and metrics that have been created for your datasource. Make sure that the name of the aggregators you use in your query match one of these metrics. Also make sure that the query interval you specify match a valid time range where data exists. Note: the broker endpoint will only return valid results on historical segments and not segments served by real-time nodes. ## How can I Reindex existing data in Druid with schema changes? @@ -50,6 +63,9 @@ To do this use the IngestSegmentFirehose and run an indexer task. The IngestSegm Typically the above will be run as a batch job to say everyday feed in a chunk of data and aggregate it. +## Real-time ingestion seems to be stuck + +There are a few ways this can occur. Druid will throttle ingestion to prevent out of memory problems if the intermediate persists are taking too long or if hand-off is taking too long. If your node logs indicate certain columns are taking a very long time to build (for example, if your segment granularity is hourly, but creating a single column takes 30 minutes), you should re-evaluate your configuration or scale up your real-time ingestion. ## More information diff --git a/docs/content/Kafka-Eight.md b/docs/content/Kafka-Eight.md index d8b3c80232e..175c71a65c6 100644 --- a/docs/content/Kafka-Eight.md +++ b/docs/content/Kafka-Eight.md @@ -8,9 +8,9 @@ The previous examples are for Kafka 7. To support Kafka 8, a couple changes need - Update realtime node's configs for Kafka 8 extensions - e.g. - - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-seven:0.6.156",...]` + - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-seven:0.6.160",...]` - becomes - - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-eight:0.6.156",...]` + - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-eight:0.6.160",...]` - Update realtime task config for changed keys - `firehose.type`, `plumber.rejectionPolicyFactory`, and all of `firehose.consumerProps` changes. diff --git a/docs/content/Production-Cluster-Configuration.md b/docs/content/Production-Cluster-Configuration.md index d8ed57ab048..21190792663 100644 --- a/docs/content/Production-Cluster-Configuration.md +++ b/docs/content/Production-Cluster-Configuration.md @@ -57,7 +57,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/overlord -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.160"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -139,7 +139,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/middlemanager -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.156","io.druid.extensions:druid-kafka-seven:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.160","io.druid.extensions:druid-kafka-seven:0.6.160"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -286,7 +286,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/historical -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.160"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Querying.md b/docs/content/Querying.md index 2b2981b96f7..dd556b2740f 100644 --- a/docs/content/Querying.md +++ b/docs/content/Querying.md @@ -66,7 +66,7 @@ The dataSource JSON field shown next identifies where to apply the query. In thi "dataSource": "randSeq", ``` -The granularity JSON field specifies the bucket size for values. It could be a built-in time interval like "second", "minute", "fifteen_minute", "thirty_minute", "hour" or "day". It can also be an expression like `{"type": "period", "period":"PT6m"}` meaning "6 minute buckets". See [Granularities](Granularities.html) for more information on the different options for this field. In this example, it is set to the special value "all" which means [bucket all data points together into the same time bucket]() +The granularity JSON field specifies the bucket size for values. It could be a built-in time interval like "second", "minute", "fifteen_minute", "thirty_minute", "hour" or "day". It can also be an expression like `{"type": "period", "period":"PT6m"}` meaning "6 minute buckets". See [Granularities](Granularities.html) for more information on the different options for this field. In this example, it is set to the special value "all" which means bucket all data points together into the same time bucket. ```javascript "granularity": "all", @@ -88,7 +88,7 @@ A groupBy also requires the JSON field "aggregations" (See [Aggregations](Aggreg ], ``` -You can also specify postAggregations, which are applied after data has been aggregated for the current granularity and dimensions bucket. See [Post Aggregations](Post Aggregations.html) for a detailed description. In the rand example, an arithmetic type operation (division, as specified by "fn") is performed with the result "name" of "avg_random". The "fields" field specifies the inputs from the aggregation stage to this expression. Note that identifiers corresponding to "name" JSON field inside the type "fieldAccess" are required but not used outside this expression, so they are prefixed with "dummy" for clarity: +You can also specify postAggregations, which are applied after data has been aggregated for the current granularity and dimensions bucket. See [Post Aggregations](Post-aggregations.html) for a detailed description. In the rand example, an arithmetic type operation (division, as specified by "fn") is performed with the result "name" of "avg_random". The "fields" field specifies the inputs from the aggregation stage to this expression. Note that identifiers corresponding to "name" JSON field inside the type "fieldAccess" are required but not used outside this expression, so they are prefixed with "dummy" for clarity: ```javascript "postAggregations": [{ @@ -127,13 +127,13 @@ Properties shared by all query types |timeseries, topN, groupBy, search|filter|Specifies the filter (the "WHERE" clause in SQL) for the query. See [Filters](Filters.html)|no| |timeseries, topN, groupBy, search|granularity|the timestamp granularity to bucket results into (i.e. "hour"). See [Granularities](Granularities.html) for more information.|no| |timeseries, topN, groupBy|aggregations|aggregations that combine values in a bucket. See [Aggregations](Aggregations.html).|yes| -|timeseries, topN, groupBy|postAggregations|aggregations of aggregations. See [Post Aggregations](Post Aggregations.html).|yes| +|timeseries, topN, groupBy|postAggregations|aggregations of aggregations. See [Post Aggregations](Post-aggregations.html).|yes| |groupBy|dimensions|constrains the groupings; if empty, then one value per time granularity bucket|yes| |search|limit|maximum number of results (default is 1000), a system-level maximum can also be set via `com.metamx.query.search.maxSearchLimit`|no| |search|searchDimensions|Dimensions to apply the search query to. If not specified, it will search through all dimensions.|no| |search|query|The query portion of the search query. This is essentially a predicate that specifies if something matches.|yes| -Query Context +Query Context ------------- |property |default | description | diff --git a/docs/content/Realtime-Config.md b/docs/content/Realtime-Config.md index 6cb566fac16..92ab9382f81 100644 --- a/docs/content/Realtime-Config.md +++ b/docs/content/Realtime-Config.md @@ -27,7 +27,7 @@ druid.host=localhost druid.service=realtime druid.port=8083 -druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.160"] druid.zk.service.host=localhost @@ -76,7 +76,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/realtime -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.156","io.druid.extensions:druid-kafka-seven:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.160","io.druid.extensions:druid-kafka-seven:0.6.160"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Recommendations.md b/docs/content/Recommendations.md new file mode 100644 index 00000000000..bf764ffe6c2 --- /dev/null +++ b/docs/content/Recommendations.md @@ -0,0 +1,35 @@ +--- +layout: doc_page +--- + +Recommendations +=============== + +# Use UTC Timezone + +We recommend using UTC timezone for all your events and across on your nodes, not just for Druid, but for all data infrastructure. This can greatly mitigate potential query problems with inconsistent timezones. + +# Use Lowercase Strings for Column Names + +Druid is not perfect in how it handles mix-cased dimension and metric names. This will hopefully change very soon but for the time being, lower casing your column names is recommended. + +# SSDs + +SSDs are highly recommended for historical and real-time nodes if you are not running a cluster that is entirely in memory. SSDs can greatly mitigate the time required to page data in and out of memory. + +# Provide Columns Names in Lexicographic Order + +Although Druid supports schema-less ingestion of dimensions, because of [https://github.com/metamx/druid/issues/658](https://github.com/metamx/druid/issues/658), you may sometimes get bigger segments than necessary. To ensure segments are as compact as possible, providing dimension names in lexicographic order is recommended. + + +# Use Timeseries and TopN Queries Instead of GroupBy Where Possible + +Timeseries and TopN queries are much more optimized and significantly faster than groupBy queries for their designed use cases. Issuing multiple topN or timeseries queries from your application can potentially be more efficient than a single groupBy query. + +# Read FAQs + +You should read common problems people have here: + +1) [Ingestion-FAQ](Ingestion-FAQ.html) + +2) [Performance-FAQ](Performance-FAQ.html) \ No newline at end of file diff --git a/docs/content/Router.md b/docs/content/Router.md new file mode 100644 index 00000000000..064bf1d646d --- /dev/null +++ b/docs/content/Router.md @@ -0,0 +1,133 @@ +--- +layout: doc_page +--- + +Router Node +=========== + +You should only ever need the router node if you have a Druid cluster well into the terabyte range. The router node can be used to route queries to different broker nodes. By default, the broker routes queries based on how [Rules](Rules.html) are set up. For example, if 1 month of recent data is loaded into a `hot` cluster, queries that fall within the recent month can be routed to a dedicated set of brokers. Queries outside this range are routed to another set of brokers. This set up provides query isolation such that queries for more important data are not impacted by queries for less important data. + +Running +------- + +``` +io.druid.cli.Main server router +``` + +Example Production Configuration +-------------------------------- + +In this example, we have two tiers in our production cluster: `hot` and `_default_tier`. Queries for the `hot` tier are routed through the `broker-hot` set of brokers, and queries for the `_default_tier` are routed through the `broker-cold` set of brokers. If any exceptions or network problems occur, queries are routed to the `broker-cold` set of brokers. In our example, we are running with a c3.2xlarge EC2 node. + +JVM settings: + +``` +-server +-Xmx13g +-Xms13g +-XX:NewSize=256m +-XX:MaxNewSize=256m +-XX:+UseConcMarkSweepGC +-XX:+PrintGCDetails +-XX:+PrintGCTimeStamps +-XX:+UseLargePages +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=/mnt/galaxy/deploy/current/ +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=/mnt/tmp + +-Dcom.sun.management.jmxremote.port=17071 +-Dcom.sun.management.jmxremote.authenticate=false +-Dcom.sun.management.jmxremote.ssl=false +``` + +Runtime.properties: + +``` +druid.host=#{IP_ADDR}:8080 +druid.port=8080 +druid.service=druid/prod/router + +druid.extensions.remoteRepositories=[] +druid.extensions.localRepository=lib +druid.extensions.coordinates=["io.druid.extensions:druid-histogram:0.6.160"] + +druid.zk.service.host=#{ZK_IPs} +druid.zk.paths.base=/druid/prod + +druid.discovery.curator.path=/prod/discovery + +druid.processing.numThreads=1 +druid.router.defaultBrokerServiceName=druid:prod:broker-cold +druid.router.coordinatorServiceName=druid:prod:coordinator +druid.router.tierToBrokerMap={"hot":"druid:prod:broker-hot","_default_tier":"druid:prod:broker-cold"} +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M + +druid.server.http.numThreads=100 + +druid.request.logging.type=emitter +druid.request.logging.feed=druid_requests + +druid.monitoring.monitors=["com.metamx.metrics.SysMonitor","com.metamx.metrics.JvmMonitor"] + +druid.emitter=http +druid.emitter.http.recipientBaseUrl=#{URL} + +druid.curator.compress=true +``` + +Runtime Configuration +--------------------- + +The router module uses several of the default modules in [Configuration](Configuration.html) and has the following set of configurations as well: + +|Property|Possible Values|Description|Default| +|--------|---------------|-----------|-------| +|`druid.router.defaultBrokerServiceName`|Any string.|The default broker to connect to in case service discovery fails.|"". Must be set.| +|`druid.router.tierToBrokerMap`|An ordered JSON map of tiers to broker names. The priority of brokers is based on the ordering.|Queries for a certain tier of data are routed to their appropriate broker.|{"_default": ""}| +|`druid.router.defaultRule`|Any string.|The default rule for all datasources.|"_default"| +|`druid.router.rulesEndpoint`|Any string.|The coordinator endpoint to extract rules from.|"/druid/coordinator/v1/rules"| +|`druid.router.coordinatorServiceName`|Any string.|The service discovery name of the coordinator.|null. Must be set.| +|`druid.router.pollPeriod`|Any ISO8601 duration.|How often to poll for new rules.|PT1M| +|`druid.router.strategies`|An ordered JSON array of objects.|All custom strategies to use for routing.|[{"type":"timeBoundary"},{"type":"priority"}]| + +Router Strategies +----------------- +The router has a configurable list of strategies for how it selects which brokers to route queries to. The order of the strategies matter because as soon as a strategy condition is matched, a broker is selected. + +### timeBoundary + +```json +{ + "type":"timeBoundary" +} +``` + +Including this strategy means all timeBoundary queries are always routed to the highest priority broker. + +### priority + +```json +{ + "type":"priority", + "minPriority":0, + "maxPriority":1 +} +``` + +Queries with a priority set to less than minPriority are routed to the lowest priority broker. Queries with priority set to greater than maxPriority are routed to the highest priority broker. By default, minPriority is 0 and maxPriority is 1. Using these default values, if a query with priority 0 (the default query priority is 0) is sent, the query skips the priority selection logic. + +### javascript + +Allows defining arbitrary routing rules using a JavaScript function. The function is passed the configuration and the query to be executed, and returns the tier it should be routed to, or null for the default tier. + +*Example*: a function that return the highest priority broker unless the given query has more than two aggregators. + +```json +{ + "type" : "javascript", + "function" : "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" +} +``` diff --git a/docs/content/Rule-Configuration.md b/docs/content/Rule-Configuration.md index bf8b8a9792d..c25c9e62b70 100644 --- a/docs/content/Rule-Configuration.md +++ b/docs/content/Rule-Configuration.md @@ -2,12 +2,34 @@ layout: doc_page --- # Configuring Rules for Coordinator Nodes + +Rules indicate how segments should be assigned to different historical node tiers and how many replicas of a segment should exist in each tier. Rules may also indicate when segments should be dropped entirely from the cluster. The coordinator loads a set of rules from the metadata storage. Rules may be specific to a certain datasource and/or a default set of rules can be configured. Rules are read in order and hence the ordering of rules is important. The coordinator will cycle through all available segments and match each segment with the first rule that applies. Each segment may only match a single rule. + Note: It is recommended that the coordinator console is used to configure rules. However, the coordinator node does have HTTP endpoints to programmatically configure rules. + Load Rules ---------- -Load rules indicate how many replicants of a segment should exist in a server tier. +Load rules indicate how many replicas of a segment should exist in a server tier. + +### Forever Load Rule + +Forever load rules are of the form: + +```json +{ + "type" : "loadForever", + "tieredReplicants": { + "hot": 1, + "_default_tier" : 1 + } +} +``` + +* `type` - this should always be "loadByInterval" +* `tieredReplicants` - A JSON Object where the keys are the tier names and values are the number of replicas for that tier. + ### Interval Load Rule @@ -16,14 +38,17 @@ Interval load rules are of the form: ```json { "type" : "loadByInterval", - "interval" : "2012-01-01/2013-01-01", - "tier" : "hot" + "interval": "2012-01-01/2013-01-01", + "tieredReplicants": { + "hot": 1, + "_default_tier" : 1 + } } ``` * `type` - this should always be "loadByInterval" * `interval` - A JSON Object representing ISO-8601 Intervals -* `tier` - the configured historical node tier +* `tieredReplicants` - A JSON Object where the keys are the tier names and values are the number of replicas for that tier. ### Period Load Rule @@ -33,13 +58,16 @@ Period load rules are of the form: { "type" : "loadByPeriod", "period" : "P1M", - "tier" : "hot" + "tieredReplicants": { + "hot": 1, + "_default_tier" : 1 + } } ``` * `type` - this should always be "loadByPeriod" * `period` - A JSON Object representing ISO-8601 Periods -* `tier` - the configured historical node tier +* `tieredReplicants` - A JSON Object where the keys are the tier names and values are the number of replicas for that tier. The interval of a segment will be compared against the specified period. The rule matches if the period overlaps the interval. @@ -48,6 +76,21 @@ Drop Rules Drop rules indicate when segments should be dropped from the cluster. +### Forever Drop Rule + +Forever drop rules are of the form: + +```json +{ + "type" : "dropForever" +} +``` + +* `type` - this should always be "dropByPeriod" + +All segments that match this rule are dropped from the cluster. + + ### Interval Drop Rule Interval drop rules are of the form: diff --git a/docs/content/Simple-Cluster-Configuration.md b/docs/content/Simple-Cluster-Configuration.md index 788f2d65af8..c25f349f0b0 100644 --- a/docs/content/Simple-Cluster-Configuration.md +++ b/docs/content/Simple-Cluster-Configuration.md @@ -28,7 +28,7 @@ Configuration: -Ddruid.zk.service.host=localhost --Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.156"] +-Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.160"] -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid -Ddruid.db.connector.user=druid diff --git a/docs/content/Thanks.md b/docs/content/Thanks.md deleted file mode 100644 index 97ec7e0904a..00000000000 --- a/docs/content/Thanks.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -layout: doc_page ---- - -YourKit supports the Druid open source projects with its -full-featured Java Profiler. -YourKit, LLC is the creator of innovative and intelligent tools for profiling -Java and .NET applications. Take a look at YourKit's software products: -YourKit Java -Profiler and -YourKit .NET -Profiler. diff --git a/docs/content/Tutorial:-A-First-Look-at-Druid.md b/docs/content/Tutorial:-A-First-Look-at-Druid.md index a0c89c0fafb..fadc7fbc963 100644 --- a/docs/content/Tutorial:-A-First-Look-at-Druid.md +++ b/docs/content/Tutorial:-A-First-Look-at-Druid.md @@ -49,7 +49,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu ### Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.156-bin.tar.gz). Download this file to a directory of your choosing. +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.160-bin.tar.gz). Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -60,7 +60,7 @@ tar -zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.156 +cd druid-services-0.6.160 ``` You should see a bunch of files: diff --git a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md index 399af3cc786..eed3030c71e 100644 --- a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md +++ b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md @@ -91,7 +91,7 @@ druid.service=overlord druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.160"] druid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid druid.db.connector.user=druid diff --git a/docs/content/Tutorial:-The-Druid-Cluster.md b/docs/content/Tutorial:-The-Druid-Cluster.md index 13483109143..7b21b72aa69 100644 --- a/docs/content/Tutorial:-The-Druid-Cluster.md +++ b/docs/content/Tutorial:-The-Druid-Cluster.md @@ -13,7 +13,7 @@ In this tutorial, we will set up other types of Druid nodes and external depende If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first. -You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.156-bin.tar.gz) +You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.160-bin.tar.gz) and untar the contents within by issuing: @@ -149,7 +149,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.160"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b @@ -240,7 +240,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.156","io.druid.extensions:druid-kafka-seven:0.6.156"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.160","io.druid.extensions:druid-kafka-seven:0.6.160"] # Change this config to db to hand off to the rest of the Druid cluster druid.publish.type=noop diff --git a/docs/content/Tutorial:-Webstream.md b/docs/content/Tutorial:-Webstream.md index d28dc27923c..5739f625a5e 100644 --- a/docs/content/Tutorial:-Webstream.md +++ b/docs/content/Tutorial:-Webstream.md @@ -37,7 +37,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu h3. Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.156-bin.tar.gz) +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.160-bin.tar.gz) Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -48,7 +48,7 @@ tar zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.156 +cd druid-services-0.6.160 ``` You should see a bunch of files: diff --git a/docs/content/Twitter-Tutorial.md b/docs/content/Twitter-Tutorial.md index db0eab430c6..844788a33b2 100644 --- a/docs/content/Twitter-Tutorial.md +++ b/docs/content/Twitter-Tutorial.md @@ -9,7 +9,7 @@ There are two ways to setup Druid: download a tarball, or build it from source. # Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.156-bin.tar.gz). +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.160-bin.tar.gz). Download this bad boy to a directory of your choosing. You can extract the awesomeness within by issuing: diff --git a/docs/content/index.md b/docs/content/index.md index 529a2325436..3c236cc81be 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -37,17 +37,6 @@ When Druid? * You want to do your analysis on data as it’s happening (in real-time) * You need a data store that is always available, 24x7x365, and years into the future. - -Not Druid? ----------- - -* The amount of data you have can easily be handled by MySQL -* You're querying for individual entries or doing lookups (not analytics) -* Batch ingestion is good enough -* Canned queries are good enough -* Downtime is no big deal - - Druid vs… ---------- @@ -60,7 +49,7 @@ Druid vs… About This Page ---------- -The data store world is vast, confusing and constantly in flux. This page is meant to help potential evaluators decide whether Druid is a good fit for the problem one needs to solve. If anything about it is incorrect please provide that feedback on the mailing list or via some other means so we can fix it. +The data infrastructure world is vast, confusing and constantly in flux. This page is meant to help potential evaluators decide whether Druid is a good fit for the problem one needs to solve. If anything about it is incorrect please provide that feedback on the mailing list or via some other means so we can fix it. diff --git a/docs/content/toc.textile b/docs/content/toc.textile index 437e383b760..29c08ab4d0e 100644 --- a/docs/content/toc.textile +++ b/docs/content/toc.textile @@ -17,7 +17,9 @@ h2. Getting Started h2. Booting a Druid Cluster * "Simple Cluster Configuration":Simple-Cluster-Configuration.html * "Production Cluster Configuration":Production-Cluster-Configuration.html +* "Production Hadoop Configuration":Hadoop-Configuration.html * "Rolling Cluster Updates":Rolling-Updates.html +* "Recommendations":Recommendations.html h2. Configuration * "Common Configuration":Configuration.html @@ -84,6 +86,7 @@ h2. Experimental * "Geographic Queries":./GeographicQueries.html * "Select Query":./SelectQuery.html * "Approximate Histograms and Quantiles":./ApproxHisto.html +* "Router node":./Router.html h2. Development * "Versioning":./Versioning.html @@ -91,4 +94,4 @@ h2. Development * "Libraries":./Libraries.html h2. Misc -* "Thanks":./Thanks.html +* "Thanks":/thanks.html diff --git a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java index b44fbd524e1..6f481d89161 100644 --- a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java +++ b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java @@ -1019,8 +1019,6 @@ public class ApproximateHistogram * @param count current size of the heap * @param heapIndex index of the item to be deleted * @param values values stored in the heap - * - * @return */ private static int heapDelete(int[] heap, int[] reverseIndex, int count, int heapIndex, float[] values) { diff --git a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index 458486423f9..c704c72ba2f 100644 --- a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -29,8 +29,14 @@ import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.Aggregators; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.hyperloglog.HyperLogLogCollector; +import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregator; +import io.druid.query.aggregation.hyperloglog.HyperUniquesBufferAggregator; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.ObjectColumnSelector; import org.apache.commons.codec.binary.Base64; import java.nio.ByteBuffer; @@ -113,7 +119,7 @@ public class ApproximateHistogramAggregatorFactory implements AggregatorFactory @Override public AggregatorFactory getCombiningFactory() { - return new ApproximateHistogramAggregatorFactory(name, name, resolution, numBuckets, lowerLimit, upperLimit); + return new ApproximateHistogramFoldingAggregatorFactory(name, name, resolution, numBuckets, lowerLimit, upperLimit); } @Override diff --git a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java index 04dc43a804b..50a9e6ba973 100644 --- a/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java +++ b/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java @@ -76,7 +76,8 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis }; } - if (ApproximateHistogram.class.isAssignableFrom(selector.classOfObject())) { + final Class cls = selector.classOfObject(); + if (cls.equals(Object.class) || ApproximateHistogram.class.isAssignableFrom(cls)) { return new ApproximateHistogramFoldingAggregator( name, selector, @@ -89,7 +90,7 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis throw new IAE( "Incompatible type for metric[%s], expected a ApproximateHistogram, got a %s", fieldName, - selector.classOfObject() + cls ); } @@ -117,14 +118,15 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis }; } - if (ApproximateHistogram.class.isAssignableFrom(selector.classOfObject())) { + final Class cls = selector.classOfObject(); + if (cls.equals(Object.class) || ApproximateHistogram.class.isAssignableFrom(cls)) { return new ApproximateHistogramFoldingBufferAggregator(selector, resolution, lowerLimit, upperLimit); } throw new IAE( "Incompatible type for metric[%s], expected a ApproximateHistogram, got a %s", fieldName, - selector.classOfObject() + cls ); } diff --git a/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java b/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java index 384e6eb1fbf..6785ceb9c10 100644 --- a/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java +++ b/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java @@ -84,5 +84,12 @@ public class Histogram return result; } - -} \ No newline at end of file + @Override + public String toString() + { + return "Histogram{" + + "breaks=" + Arrays.toString(breaks) + + ", counts=" + Arrays.toString(counts) + + '}'; + } +} diff --git a/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java new file mode 100644 index 00000000000..48814642f5b --- /dev/null +++ b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java @@ -0,0 +1,217 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation.histogram; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.druid.collections.StupidPool; +import io.druid.data.input.Row; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryEngine; +import io.druid.query.groupby.GroupByQueryQueryToolChest; +import io.druid.query.groupby.GroupByQueryRunnerFactory; +import io.druid.query.groupby.GroupByQueryRunnerTestHelper; +import io.druid.query.groupby.orderby.DefaultLimitSpec; +import io.druid.query.groupby.orderby.OrderByColumnSpec; +import io.druid.segment.TestHelper; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + */ +@RunWith(Parameterized.class) +public class ApproximateHistogramGroupByQueryTest +{ + private final QueryRunner runner; + private GroupByQueryRunnerFactory factory; + + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + final ObjectMapper mapper = new DefaultObjectMapper(); + final StupidPool pool = new StupidPool( + new Supplier() + { + @Override + public ByteBuffer get() + { + return ByteBuffer.allocate(1024 * 1024); + } + } + ); + + final GroupByQueryConfig config = new GroupByQueryConfig(); + config.setMaxIntermediateRows(10000); + + final Supplier configSupplier = Suppliers.ofInstance(config); + final GroupByQueryEngine engine = new GroupByQueryEngine(configSupplier, pool); + + final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( + engine, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, + configSupplier, + new GroupByQueryQueryToolChest(configSupplier, mapper, engine, pool), + pool + ); + + GroupByQueryConfig singleThreadedConfig = new GroupByQueryConfig() + { + @Override + public boolean isSingleThreaded() + { + return true; + } + }; + singleThreadedConfig.setMaxIntermediateRows(10000); + + final Supplier singleThreadedConfigSupplier = Suppliers.ofInstance(singleThreadedConfig); + final GroupByQueryEngine singleThreadEngine = new GroupByQueryEngine(singleThreadedConfigSupplier, pool); + + final GroupByQueryRunnerFactory singleThreadFactory = new GroupByQueryRunnerFactory( + singleThreadEngine, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, + singleThreadedConfigSupplier, + new GroupByQueryQueryToolChest(singleThreadedConfigSupplier, mapper, singleThreadEngine, pool), + pool + ); + + + Function function = new Function() + { + @Override + public Object apply(@Nullable Object input) + { + return new Object[]{factory, ((Object[]) input)[0]}; + } + }; + + return Lists.newArrayList( + Iterables.concat( + Iterables.transform( + QueryRunnerTestHelper.makeQueryRunners(factory), + function + ), + Iterables.transform( + QueryRunnerTestHelper.makeQueryRunners(singleThreadFactory), + function + ) + ) + ); + } + + public ApproximateHistogramGroupByQueryTest(GroupByQueryRunnerFactory factory, QueryRunner runner) + { + this.factory = factory; + this.runner = runner; + } + + @Test + public void testGroupByWithApproximateHistogramAgg() + { + ApproximateHistogramAggregatorFactory aggFactory = new ApproximateHistogramAggregatorFactory( + "apphisto", + "index", + 10, + 5, + Float.NEGATIVE_INFINITY, + Float.POSITIVE_INFINITY + ); + + GroupByQuery query = new GroupByQuery.Builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setGranularity(QueryRunnerTestHelper.allGran) + .setDimensions( + Arrays.asList( + new DefaultDimensionSpec( + QueryRunnerTestHelper.marketDimension, + "marKetAlias" + ) + ) + ) + .setInterval(QueryRunnerTestHelper.fullOnInterval) + .setLimitSpec( + new DefaultLimitSpec( + Lists.newArrayList( + new OrderByColumnSpec( + "marKetAlias", + OrderByColumnSpec.Direction.DESCENDING + ) + ), 1 + ) + ) + .setAggregatorSpecs( + Lists.newArrayList( + QueryRunnerTestHelper.rowsCount, + aggFactory + ) + ) + .setPostAggregatorSpecs( + Arrays.asList( + new QuantilePostAggregator("quantile", "apphisto", 0.5f) + ) + ) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "marketalias", "upfront", + "rows", 186L, + "quantile", 880.9881f, + "apphisto", + new Histogram( + new float[]{ + 214.97299194335938f, + 545.9906005859375f, + 877.0081787109375f, + 1208.0257568359375f, + 1539.0433349609375f, + 1870.06103515625f + }, + new double[]{ + 0.0, 67.53287506103516, 72.22068786621094, 31.984678268432617, 14.261756896972656 + } + ) + ) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "approx-histo"); + } +} diff --git a/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramQueryTest.java b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java similarity index 96% rename from histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramQueryTest.java rename to histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java index 0599e6aebad..bba68f5339e 100644 --- a/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramQueryTest.java +++ b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java @@ -53,18 +53,8 @@ import java.util.List; import java.util.Map; @RunWith(Parameterized.class) -public class ApproximateHistogramQueryTest +public class ApproximateHistogramTopNQueryTest { - - private final QueryRunner runner; - - public ApproximateHistogramQueryTest( - QueryRunner runner - ) - { - this.runner = runner; - } - @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { @@ -100,6 +90,15 @@ public class ApproximateHistogramQueryTest return retVal; } + private final QueryRunner runner; + + public ApproximateHistogramTopNQueryTest( + QueryRunner runner + ) + { + this.runner = runner; + } + @Test public void testTopNWithApproximateHistogramAgg() { @@ -115,7 +114,7 @@ public class ApproximateHistogramQueryTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -146,7 +145,7 @@ public class ApproximateHistogramQueryTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "total_market") + .put(QueryRunnerTestHelper.marketDimension, "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -177,7 +176,7 @@ public class ApproximateHistogramQueryTest ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "upfront") + .put(QueryRunnerTestHelper.marketDimension, "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -208,7 +207,7 @@ public class ApproximateHistogramQueryTest ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "spot") + .put(QueryRunnerTestHelper.marketDimension, "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java index fcf24454118..8c6eaf8d528 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java @@ -41,6 +41,7 @@ import com.metamx.common.logger.Logger; import io.druid.common.utils.JodaUtils; import io.druid.data.input.InputRow; import io.druid.data.input.impl.StringInputRowParser; +import io.druid.granularity.QueryGranularity; import io.druid.guice.GuiceInjectors; import io.druid.guice.JsonConfigProvider; import io.druid.guice.annotations.Self; @@ -171,6 +172,7 @@ public class HadoopDruidIndexerConfig private volatile PathSpec pathSpec; private volatile Map shardSpecLookups = Maps.newHashMap(); private volatile Map hadoopShardSpecLookup = Maps.newHashMap(); + private final QueryGranularity rollupGran; @JsonCreator public HadoopDruidIndexerConfig( @@ -202,6 +204,7 @@ public class HadoopDruidIndexerConfig hadoopShardSpecLookup.put(hadoopyShardSpec.getActualSpec(), hadoopyShardSpec); } } + this.rollupGran = schema.getDataSchema().getGranularitySpec().getQueryGranularity(); } @JsonProperty @@ -325,7 +328,7 @@ public class HadoopDruidIndexerConfig return Optional.absent(); } - final ShardSpec actualSpec = shardSpecLookups.get(timeBucket.get().getStart()).getShardSpec(inputRow); + final ShardSpec actualSpec = shardSpecLookups.get(timeBucket.get().getStart()).getShardSpec(rollupGran.truncate(inputRow.getTimestampFromEpoch()), inputRow); final HadoopyShardSpec hadoopyShardSpec = hadoopShardSpecLookup.get(actualSpec); return Optional.of( diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/rollup/DataRollupSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/rollup/DataRollupSpec.java index 9f5048559d1..3f0fd29736e 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/rollup/DataRollupSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/rollup/DataRollupSpec.java @@ -20,6 +20,7 @@ package io.druid.indexer.rollup; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.api.client.util.Lists; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; @@ -34,7 +35,7 @@ import java.util.List; public class DataRollupSpec { @JsonProperty - public List aggs; + public List aggs = Lists.newArrayList(); @JsonProperty public QueryGranularity rollupGranularity = QueryGranularity.NONE; diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java index ee8fa9315c5..9149a127fda 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -20,15 +20,34 @@ package io.druid.indexer; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Lists; +import com.google.common.base.Optional; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.metamx.common.Granularity; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.impl.JSONDataSpec; +import io.druid.data.input.impl.TimestampSpec; +import io.druid.granularity.QueryGranularity; +import io.druid.indexer.rollup.DataRollupSpec; import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.indexing.DataSchema; +import io.druid.segment.indexing.granularity.UniformGranularitySpec; +import io.druid.timeline.partition.HashBasedNumberedShardSpec; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.joda.time.DateTime; +import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; +import java.util.Arrays; +import java.util.List; + /** */ public class HadoopDruidIndexerConfigTest @@ -125,4 +144,72 @@ public class HadoopDruidIndexerConfigTest ); } + + @Test + public void testHashedBucketSelection() { + List specs = Lists.newArrayList(); + final int partitionCount = 10; + for (int i = 0; i < partitionCount; i++) { + specs.add(new HadoopyShardSpec(new HashBasedNumberedShardSpec(i, partitionCount, new DefaultObjectMapper()), i)); + } + // Backwards compatibility + DataRollupSpec rollupSpec = new DataRollupSpec(); + rollupSpec.rollupGranularity = QueryGranularity.MINUTE; + + HadoopIngestionSpec spec = new HadoopIngestionSpec( + null, null, null, + "foo", + new TimestampSpec("timestamp", "auto"), + new JSONDataSpec(ImmutableList.of("foo"), null), + new UniformGranularitySpec( + Granularity.HOUR, + QueryGranularity.MINUTE, + ImmutableList.of(new Interval("2010-01-01/P1D")), + Granularity.HOUR + ), + null, + null, + null, + null, + null, + false, + true, + ImmutableMap.of(new DateTime("2010-01-01T01:00:00"), specs), + false, + rollupSpec, + null, + false, + ImmutableMap.of("foo", "bar"), + false, + null, + null, + null, + null, + null, + null + ); + HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSchema(spec); + final List dims = Arrays.asList("diM1", "dIM2"); + final ImmutableMap values = ImmutableMap.of( + "Dim1", + "1", + "DiM2", + "2", + "dim1", + "3", + "dim2", + "4" + ); + final long timestamp = new DateTime("2010-01-01T01:00:01").getMillis(); + final Bucket expectedBucket = config.getBucket(new MapBasedInputRow(timestamp, dims, values)).get(); + final long nextBucketTimestamp = QueryGranularity.MINUTE.next(QueryGranularity.MINUTE.truncate(timestamp)); + // check that all rows having same set of dims and truncated timestamp hash to same bucket + for (int i = 0; timestamp + i < nextBucketTimestamp; i++) { + Assert.assertEquals( + expectedBucket.partitionNum, + config.getBucket(new MapBasedInputRow(timestamp + i, dims, values)).get().partitionNum + ); + } + + } } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index 058b3fd027e..d1495e6b6d2 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -411,14 +411,14 @@ public class IndexTask extends AbstractFixedIntervalTask final int myRowFlushBoundary = rowFlushBoundary > 0 ? rowFlushBoundary : toolbox.getConfig().getDefaultRowFlushBoundary(); - + final QueryGranularity rollupGran = ingestionSchema.getDataSchema().getGranularitySpec().getQueryGranularity(); try { plumber.startJob(); while (firehose.hasMore()) { final InputRow inputRow = firehose.nextRow(); - if (shouldIndex(shardSpec, interval, inputRow)) { + if (shouldIndex(shardSpec, interval, inputRow, rollupGran)) { int numRows = plumber.add(inputRow); if (numRows == -1) { throw new ISE( @@ -473,13 +473,15 @@ public class IndexTask extends AbstractFixedIntervalTask * * @return true or false */ - private boolean shouldIndex( + private static boolean shouldIndex( final ShardSpec shardSpec, final Interval interval, - final InputRow inputRow + final InputRow inputRow, + final QueryGranularity rollupGran ) { - return interval.contains(inputRow.getTimestampFromEpoch()) && shardSpec.isInChunk(inputRow); + return interval.contains(inputRow.getTimestampFromEpoch()) + && shardSpec.isInChunk(rollupGran.truncate(inputRow.getTimestampFromEpoch()), inputRow); } public static class IndexIngestionSpec extends IngestionSpec diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java index f657b6a2435..f78f576de67 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java @@ -32,7 +32,7 @@ public class ImmutableZkWorker { private final Worker worker; private final int currCapacityUsed; - private final Set availabilityGroups; + private final ImmutableSet availabilityGroups; public ImmutableZkWorker(Worker worker, int currCapacityUsed, Set availabilityGroups) { diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java index e9bba479c64..e382c6dee3c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java @@ -164,7 +164,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer @Override public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception { - Worker worker; + final Worker worker; switch (event.getType()) { case CHILD_ADDED: worker = jsonMapper.readValue( @@ -198,6 +198,14 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer } ); break; + case CHILD_UPDATED: + worker = jsonMapper.readValue( + event.getData().getData(), + Worker.class + ); + updateWorker(worker); + break; + case CHILD_REMOVED: worker = jsonMapper.readValue( event.getData().getData(), @@ -745,6 +753,24 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer } } + /** + * We allow workers to change their own capacities and versions. They cannot change their own hosts or ips without + * dropping themselves and re-announcing. + */ + private void updateWorker(final Worker worker) + { + final ZkWorker zkWorker = zkWorkers.get(worker.getHost()); + if (zkWorker != null) { + log.info("Worker[%s] updated its announcement from[%s] to[%s].", worker.getHost(), zkWorker.getWorker(), worker); + zkWorker.setWorker(worker); + } else { + log.warn( + "WTF, worker[%s] updated its announcement but we didn't have a ZkWorker for it. Ignoring.", + worker.getHost() + ); + } + } + /** * When a ephemeral worker node disappears from ZK, incomplete running tasks will be retried by * the logic in the status listener. We still have to make sure there are no tasks assigned diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java index abc4da0ad57..54b09da2b29 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java @@ -22,11 +22,11 @@ package io.druid.indexing.overlord; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import io.druid.indexing.common.task.Task; import io.druid.indexing.worker.TaskAnnouncement; import io.druid.indexing.worker.Worker; import org.apache.curator.framework.recipes.cache.ChildData; @@ -46,15 +46,15 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ZkWorker implements Closeable { - private final Worker worker; private final PathChildrenCache statusCache; private final Function cacheConverter; + private AtomicReference worker; private AtomicReference lastCompletedTaskTime = new AtomicReference(new DateTime()); public ZkWorker(Worker worker, PathChildrenCache statusCache, final ObjectMapper jsonMapper) { - this.worker = worker; + this.worker = new AtomicReference<>(worker); this.statusCache = statusCache; this.cacheConverter = new Function() { @@ -84,7 +84,7 @@ public class ZkWorker implements Closeable @JsonProperty("worker") public Worker getWorker() { - return worker; + return worker.get(); } @JsonProperty("runningTasks") @@ -137,30 +137,28 @@ public class ZkWorker implements Closeable return getRunningTasks().containsKey(taskId); } - public boolean isAtCapacity() - { - return getCurrCapacityUsed() >= worker.getCapacity(); - } - public boolean isValidVersion(String minVersion) { - return worker.getVersion().compareTo(minVersion) >= 0; + return worker.get().getVersion().compareTo(minVersion) >= 0; } - public boolean canRunTask(Task task) + public void setWorker(Worker newWorker) { - return (worker.getCapacity() - getCurrCapacityUsed() >= task.getTaskResource().getRequiredCapacity() - && !getAvailabilityGroups().contains(task.getTaskResource().getAvailabilityGroup())); + final Worker oldWorker = worker.get(); + Preconditions.checkArgument(newWorker.getHost().equals(oldWorker.getHost()), "Cannot change Worker host"); + Preconditions.checkArgument(newWorker.getIp().equals(oldWorker.getIp()), "Cannot change Worker ip"); + + worker.set(newWorker); } public void setLastCompletedTaskTime(DateTime completedTaskTime) { - lastCompletedTaskTime.getAndSet(completedTaskTime); + lastCompletedTaskTime.set(completedTaskTime); } public ImmutableZkWorker toImmutable() { - return new ImmutableZkWorker(worker, getCurrCapacityUsed(), getAvailabilityGroups()); + return new ImmutableZkWorker(worker.get(), getCurrCapacityUsed(), getAvailabilityGroups()); } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index 11c7c85c639..b5f86ea7494 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -361,6 +361,29 @@ public class RemoteTaskRunnerTest Assert.assertEquals(TaskStatus.Status.FAILED, status.getStatusCode()); } + @Test + public void testWorkerDisabled() throws Exception + { + doSetup(); + final ListenableFuture result = remoteTaskRunner.run(task); + + Assert.assertTrue(taskAnnounced(task.getId())); + mockWorkerRunningTask(task); + Assert.assertTrue(workerRunningTask(task.getId())); + + // Disable while task running + disableWorker(); + + // Continue test + mockWorkerCompleteSuccessfulTask(task); + Assert.assertTrue(workerCompletedTask(result)); + Assert.assertEquals(task.getId(), result.get().getId()); + Assert.assertEquals(TaskStatus.Status.SUCCESS, result.get().getStatusCode()); + + // Confirm RTR thinks the worker is disabled. + Assert.assertEquals("", Iterables.getOnlyElement(remoteTaskRunner.getWorkers()).getWorker().getVersion()); + } + private void doSetup() throws Exception { makeWorker(); @@ -405,6 +428,14 @@ public class RemoteTaskRunnerTest ); } + private void disableWorker() throws Exception + { + cf.setData().forPath( + announcementsPath, + jsonMapper.writeValueAsBytes(new Worker(worker.getHost(), worker.getIp(), worker.getCapacity(), "")) + ); + } + private boolean taskAnnounced(final String taskId) { return pathExists(joiner.join(tasksPath, taskId)); diff --git a/pom.xml b/pom.xml index 189a52084b1..1ca0ace022d 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,8 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid druid @@ -39,7 +40,7 @@ UTF-8 - 0.26.7 + 0.26.9 2.6.0 0.2.16 @@ -89,7 +90,7 @@ com.metamx bytebuffer-collections - 0.0.2 + 0.0.4 com.metamx @@ -194,7 +195,7 @@ it.uniroma3.mat extendedset - 1.3.4 + 1.3.7 com.google.guava @@ -256,6 +257,11 @@ jackson-jaxrs-json-provider 2.2.3 + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-smile-provider + 2.2.3 + org.codehaus.jackson jackson-core-asl diff --git a/processing/pom.xml b/processing/pom.xml index 7ccbed8b4da..2f60f53d2bc 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -18,7 +18,8 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid druid-processing @@ -83,12 +84,12 @@ alphanum - net.jpountz.lz4 - lz4 + net.jpountz.lz4 + lz4 - org.mapdb - mapdb + org.mapdb + mapdb @@ -103,10 +104,10 @@ easymock test - - com.google.caliper - caliper - + + com.google.caliper + caliper + diff --git a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java index 908a17cb058..313ed85fc4c 100644 --- a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java @@ -26,6 +26,7 @@ import com.google.common.hash.Hashing; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.FilteredAggregatorFactory; import io.druid.query.aggregation.HistogramAggregatorFactory; import io.druid.query.aggregation.JavaScriptAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; @@ -68,7 +69,8 @@ public class AggregatorsModule extends SimpleModule @JsonSubTypes.Type(name = "javascript", value = JavaScriptAggregatorFactory.class), @JsonSubTypes.Type(name = "histogram", value = HistogramAggregatorFactory.class), @JsonSubTypes.Type(name = "hyperUnique", value = HyperUniquesAggregatorFactory.class), - @JsonSubTypes.Type(name = "cardinality", value = CardinalityAggregatorFactory.class) + @JsonSubTypes.Type(name = "cardinality", value = CardinalityAggregatorFactory.class), + @JsonSubTypes.Type(name = "filtered", value = FilteredAggregatorFactory.class) }) public static interface AggregatorFactoryMixin { diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index dab0ff08aba..e5edfd3d4cf 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -48,14 +48,14 @@ import java.util.concurrent.TimeoutException; /** * A QueryRunner that combines a list of other QueryRunners and executes them in parallel on an executor. - * + *

* When using this, it is important to make sure that the list of QueryRunners provided is fully flattened. * If, for example, you were to pass a list of a Chained QueryRunner (A) and a non-chained QueryRunner (B). Imagine * A has 2 QueryRunner chained together (Aa and Ab), the fact that the Queryables are run in parallel on an * executor would mean that the Queryables are actually processed in the order - * + *

* 

A -> B -> Aa -> Ab
- * + *

* That is, the two sub queryables for A would run *after* B is run, effectively meaning that the results for B * must be fully cached in memory before the results for Aa and Ab are computed. */ @@ -114,6 +114,10 @@ public class ChainedExecutionQueryRunner implements QueryRunner @Override public ListenableFuture> apply(final QueryRunner input) { + if (input == null) { + throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening"); + } + return exec.submit( new AbstractPrioritizedCallable>(priority) { @@ -156,7 +160,7 @@ public class ChainedExecutionQueryRunner implements QueryRunner queryWatcher.registerQuery(query, futures); try { - final Number timeout = query.getContextValue("timeout", (Number)null); + final Number timeout = query.getContextValue("timeout", (Number) null); return new MergeIterable<>( ordering.nullsFirst(), timeout == null ? @@ -169,10 +173,10 @@ public class ChainedExecutionQueryRunner implements QueryRunner futures.cancel(true); throw new QueryInterruptedException("Query interrupted"); } - catch(CancellationException e) { + catch (CancellationException e) { throw new QueryInterruptedException("Query cancelled"); } - catch(TimeoutException e) { + catch (TimeoutException e) { log.info("Query timeout, cancelling pending results for query id [%s]", query.getId()); futures.cancel(true); throw new QueryInterruptedException("Query timeout"); diff --git a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java index 4a5f11a1453..5997518dddd 100644 --- a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java +++ b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java @@ -29,6 +29,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.metamx.common.ISE; import com.metamx.common.Pair; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.ResourceClosingSequence; @@ -89,9 +90,6 @@ public class GroupByParallelQueryRunner implements QueryRunner final boolean bySegment = query.getContextBySegment(false); final int priority = query.getContextPriority(0); - if (Iterables.isEmpty(queryables)) { - log.warn("No queryables found."); - } ListenableFuture> futures = Futures.allAsList( Lists.newArrayList( Iterables.transform( @@ -101,6 +99,10 @@ public class GroupByParallelQueryRunner implements QueryRunner @Override public ListenableFuture apply(final QueryRunner input) { + if (input == null) { + throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening"); + } + return exec.submit( new AbstractPrioritizedCallable(priority) { diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregator.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregator.java new file mode 100644 index 00000000000..2b78ed2cd22 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregator.java @@ -0,0 +1,86 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedInts; + +import javax.annotation.Nullable; + +public class FilteredAggregator implements Aggregator +{ + private final String name; + private final DimensionSelector dimSelector; + private final Aggregator delegate; + private final IntPredicate predicate; + + public FilteredAggregator(String name, DimensionSelector dimSelector, IntPredicate predicate, Aggregator delegate) + { + this.name = name; + this.dimSelector = dimSelector; + this.delegate = delegate; + this.predicate = predicate; + } + + @Override + public void aggregate() + { + final IndexedInts row = dimSelector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (predicate.apply(row.get(i))) { + delegate.aggregate(); + break; + } + } + } + + @Override + public void reset() + { + delegate.reset(); + } + + @Override + public Object get() + { + return delegate.get(); + } + + @Override + public float getFloat() + { + return delegate.getFloat(); + } + + @Override + public String getName() + { + return name; + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java new file mode 100644 index 00000000000..cd5685a83f7 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java @@ -0,0 +1,216 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.metamx.common.ISE; +import com.metamx.common.Pair; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; + +public class FilteredAggregatorFactory implements AggregatorFactory +{ + private static final byte CACHE_TYPE_ID = 0x9; + + private final String name; + private final AggregatorFactory delegate; + private final DimFilter filter; + + public FilteredAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("aggregator") AggregatorFactory delegate, + @JsonProperty("filter") DimFilter filter + ) + { + Preconditions.checkNotNull(delegate); + Preconditions.checkNotNull(filter); + Preconditions.checkArgument( + filter instanceof SelectorDimFilter || + (filter instanceof NotDimFilter && ((NotDimFilter) filter).getField() instanceof SelectorDimFilter), + "FilteredAggregator currently only supports filters of type 'selector' and their negation" + ); + + this.name = name; + this.delegate = delegate; + this.filter = filter; + } + + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + final Aggregator aggregator = delegate.factorize(metricFactory); + final Pair selectorPredicatePair = makeFilterPredicate( + filter, + metricFactory + ); + return new FilteredAggregator(name, selectorPredicatePair.lhs, selectorPredicatePair.rhs, aggregator); + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + final BufferAggregator aggregator = delegate.factorizeBuffered(metricFactory); + final Pair selectorPredicatePair = makeFilterPredicate( + filter, + metricFactory + ); + return new FilteredBufferAggregator(selectorPredicatePair.lhs, selectorPredicatePair.rhs, aggregator); + } + + @Override + public Comparator getComparator() + { + return delegate.getComparator(); + } + + @Override + public Object combine(Object lhs, Object rhs) + { + return delegate.combine(lhs, rhs); + } + + @Override + public AggregatorFactory getCombiningFactory() + { + return delegate.getCombiningFactory(); + } + + @Override + public Object deserialize(Object object) + { + return delegate.deserialize(object); + } + + @Override + public Object finalizeComputation(Object object) + { + return delegate.finalizeComputation(object); + } + + @JsonProperty + @Override + public String getName() + { + return name; + } + + @Override + public List requiredFields() + { + return delegate.requiredFields(); + } + + @Override + public byte[] getCacheKey() + { + byte[] filterCacheKey = filter.getCacheKey(); + byte[] aggregatorCacheKey = delegate.getCacheKey(); + return ByteBuffer.allocate(1 + filterCacheKey.length + aggregatorCacheKey.length) + .put(CACHE_TYPE_ID) + .put(filterCacheKey) + .put(aggregatorCacheKey) + .array(); + } + + @Override + public String getTypeName() + { + return delegate.getTypeName(); + } + + @Override + public int getMaxIntermediateSize() + { + return delegate.getMaxIntermediateSize(); + } + + @Override + public Object getAggregatorStartValue() + { + return delegate.getAggregatorStartValue(); + } + + @JsonProperty + public AggregatorFactory getAggregator() + { + return delegate; + } + + @JsonProperty + public DimFilter getFilter() + { + return filter; + } + + @Override + public List getRequiredColumns() + { + return delegate.getRequiredColumns(); + } + + private static Pair makeFilterPredicate( + final DimFilter dimFilter, + final ColumnSelectorFactory metricFactory + ) + { + final SelectorDimFilter selector; + if (dimFilter instanceof NotDimFilter) { + // we only support NotDimFilter with Selector filter + selector = (SelectorDimFilter) ((NotDimFilter) dimFilter).getField(); + } else if (dimFilter instanceof SelectorDimFilter) { + selector = (SelectorDimFilter) dimFilter; + } else { + throw new ISE("Unsupported DimFilter type [%d]", dimFilter.getClass()); + } + + final DimensionSelector dimSelector = metricFactory.makeDimensionSelector(selector.getDimension()); + final int lookupId = dimSelector.lookupId(selector.getValue()); + final IntPredicate predicate; + if (dimFilter instanceof NotDimFilter) { + predicate = new IntPredicate() + { + @Override + public boolean apply(int value) + { + return lookupId != value; + } + }; + } else { + predicate = new IntPredicate() + { + @Override + public boolean apply(int value) + { + return lookupId == value; + } + }; + } + return Pair.of(dimSelector, predicate); + } + +} diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/FilteredBufferAggregator.java new file mode 100644 index 00000000000..42f88138cf5 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredBufferAggregator.java @@ -0,0 +1,82 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedInts; + +import java.nio.ByteBuffer; + +public class FilteredBufferAggregator implements BufferAggregator +{ + private final DimensionSelector dimSelector; + private final IntPredicate predicate; + private final BufferAggregator delegate; + + public FilteredBufferAggregator(DimensionSelector dimSelector, IntPredicate predicate, BufferAggregator delegate) + { + this.dimSelector = dimSelector; + this.predicate = predicate; + this.delegate = delegate; + } + + @Override + public void init(ByteBuffer buf, int position) + { + delegate.init(buf, position); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + final IndexedInts row = dimSelector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (predicate.apply(row.get(i))) { + delegate.aggregate(buf, position); + break; + } + } + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return delegate.get(buf, position); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + return delegate.getLong(buf, position); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + return delegate.getFloat(buf, position); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/IntPredicate.java b/processing/src/main/java/io/druid/query/aggregation/IntPredicate.java new file mode 100644 index 00000000000..fd013004e91 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/IntPredicate.java @@ -0,0 +1,29 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.aggregation; + +/** + * can be replaced with http://docs.oracle.com/javase/8/docs/api/java/util/function/IntPredicate.html + * when druid moves to java 8. + */ +public interface IntPredicate +{ + boolean apply(int value); +} diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index b1db05a2284..071a91b3c3e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -63,12 +63,12 @@ public class CardinalityAggregatorFactory implements AggregatorFactory public CardinalityAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldNames") final List fieldNames, - @JsonProperty("byRow") final Boolean byRow + @JsonProperty("byRow") final boolean byRow ) { this.name = name; this.fieldNames = fieldNames; - this.byRow = byRow == null ? false : byRow; + this.byRow = byRow; } @Override @@ -203,6 +203,12 @@ public class CardinalityAggregatorFactory implements AggregatorFactory return fieldNames; } + @JsonProperty + public boolean isByRow() + { + return byRow; + } + @Override public byte[] getCacheKey() { diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollector.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollector.java index b93c7ace592..7ccbdac2805 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollector.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollector.java @@ -200,9 +200,6 @@ public abstract class HyperLogLogCollector implements Comparable return this; } - public Builder setInterval(Object interval) + public Builder setInterval(QuerySegmentSpec interval) + { + return setQuerySegmentSpec(interval); + } + + public Builder setInterval(List intervals) + { + return setQuerySegmentSpec(new LegacySegmentSpec(intervals)); + } + + public Builder setInterval(Interval interval) + { + return setQuerySegmentSpec(new LegacySegmentSpec(interval)); + } + + public Builder setInterval(String interval) { return setQuerySegmentSpec(new LegacySegmentSpec(interval)); } diff --git a/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java index 37ad9e1b8df..e984c0cafa8 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java @@ -49,8 +49,6 @@ public interface HavingSpec * @param row A Row of data that may contain aggregated values * * @return true if the given row satisfies the having spec. False otherwise. - * - * @see Row */ public boolean eval(Row row); diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index bf6fc138ad4..ad96280a925 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -173,7 +173,7 @@ public class DefaultLimitSpec implements LimitSpec public String apply(Row input) { // Multi-value dimensions have all been flattened at this point; - final List dimList = input.getDimension(dimension); + final List dimList = input.getDimension(dimension.toLowerCase()); return dimList.isEmpty() ? null : dimList.get(0); } } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index dca9723b6c8..7e1001afb11 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -537,6 +537,10 @@ public class IndexIO final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); switch (holder.getType()) { + case LONG: + builder.setValueType(ValueType.LONG); + builder.addSerde(new LongGenericColumnPartSerde(holder.longType, BYTE_ORDER)); + break; case FLOAT: builder.setValueType(ValueType.FLOAT); builder.addSerde(new FloatGenericColumnPartSerde(holder.floatType, BYTE_ORDER)); diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 98828642036..d5947b21cab 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -637,6 +637,7 @@ public class IndexMerger switch (type) { case LONG: metWriters.add(new LongMetricColumnSerializer(metric, v8OutDir, ioPeon)); + break; case FLOAT: metWriters.add(new FloatMetricColumnSerializer(metric, v8OutDir, ioPeon)); break; diff --git a/processing/src/main/java/io/druid/segment/MetricHolder.java b/processing/src/main/java/io/druid/segment/MetricHolder.java index c0608ffe151..f5ccc83e190 100644 --- a/processing/src/main/java/io/druid/segment/MetricHolder.java +++ b/processing/src/main/java/io/druid/segment/MetricHolder.java @@ -28,11 +28,13 @@ import com.metamx.common.guava.CloseQuietly; import io.druid.common.utils.SerializerUtils; import io.druid.segment.data.CompressedFloatsIndexedSupplier; import io.druid.segment.data.CompressedFloatsSupplierSerializer; +import io.druid.segment.data.CompressedLongsIndexedSupplier; import io.druid.segment.data.CompressedLongsSupplierSerializer; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; @@ -147,6 +149,9 @@ public class MetricHolder MetricHolder holder = new MetricHolder(metricName, typeName); switch (holder.type) { + case LONG: + holder.longType = CompressedLongsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder()); + break; case FLOAT: holder.floatType = CompressedFloatsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder()); break; @@ -174,18 +179,22 @@ public class MetricHolder public enum MetricType { + LONG, FLOAT, COMPLEX; static MetricType determineType(String typeName) { - if ("float".equalsIgnoreCase(typeName)) { + if ("long".equalsIgnoreCase(typeName)) { + return LONG; + } else if ("float".equalsIgnoreCase(typeName)) { return FLOAT; } return COMPLEX; } } + CompressedLongsIndexedSupplier longType = null; CompressedFloatsIndexedSupplier floatType = null; Indexed complexType = null; @@ -214,6 +223,12 @@ public class MetricHolder return type; } + public IndexedLongs getLongType() + { + assertType(MetricType.LONG); + return longType.get(); + } + public IndexedFloats getFloatType() { assertType(MetricType.FLOAT); @@ -228,9 +243,14 @@ public class MetricHolder public MetricHolder convertByteOrder(ByteOrder order) { + MetricHolder retVal; switch (type) { + case LONG: + retVal = new MetricHolder(name, typeName); + retVal.longType = longType.convertByteOrder(order); + return retVal; case FLOAT: - MetricHolder retVal = new MetricHolder(name, typeName); + retVal = new MetricHolder(name, typeName); retVal.floatType = floatType.convertByteOrder(order); return retVal; case COMPLEX: diff --git a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java index 35a3b03f0b2..58f57d1191a 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java @@ -179,8 +179,6 @@ public class CompressedFloatsIndexedSupplier implements Supplier /** * For testing. Do not depend on unless you like things breaking. - * - * @return */ GenericIndexed> getBaseFloatBuffers() { diff --git a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java index 56998d09886..578712d16f3 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java @@ -178,7 +178,7 @@ public class CompressedLongsIndexedSupplier implements Supplier baseLongBuffers.writeToChannel(channel); } - public CompressedLongsIndexedSupplier convertByteOrder(ByteOrder order, CompressedObjectStrategy.CompressionStrategy compression) + public CompressedLongsIndexedSupplier convertByteOrder(ByteOrder order) { return new CompressedLongsIndexedSupplier( totalSize, @@ -190,7 +190,6 @@ public class CompressedLongsIndexedSupplier implements Supplier /** * For testing. Do not use unless you like things breaking - * @return */ GenericIndexed> getBaseLongBuffers() { diff --git a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java index 36a3ad4876e..0796b60f66b 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java @@ -20,8 +20,8 @@ package io.druid.segment.data; import com.google.common.base.Throwables; -import com.metamx.common.guava.CloseQuietly; import com.google.common.collect.Maps; +import com.metamx.common.guava.CloseQuietly; import com.ning.compress.lzf.ChunkEncoder; import com.ning.compress.lzf.LZFChunk; import com.ning.compress.lzf.LZFDecoder; @@ -30,7 +30,6 @@ import io.druid.segment.CompressedPools; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.lz4.LZ4SafeDecompressor; -import net.jpountz.lz4.LZ4UnknownSizeDecompressor; import java.io.IOException; import java.nio.Buffer; diff --git a/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java index e3c5de7a60a..ce4f4736450 100644 --- a/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java +++ b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java @@ -142,10 +142,10 @@ public class SpatialDimensionRowTransformer implements Function expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -241,7 +236,7 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "rows", 26L, @@ -250,7 +245,7 @@ public class GroupByQueryRunnerTest ) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -271,7 +266,7 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "rows", 26L, @@ -280,7 +275,7 @@ public class GroupByQueryRunnerTest ) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -310,26 +305,26 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "a", "rows", 1L, "idx", 135L), - createExpectedRow("2011-04-01", "alias", "b", "rows", 1L, "idx", 118L), - createExpectedRow("2011-04-01", "alias", "e", "rows", 1L, "idx", 158L), - createExpectedRow("2011-04-01", "alias", "h", "rows", 1L, "idx", 120L), - createExpectedRow("2011-04-01", "alias", "m", "rows", 3L, "idx", 2870L), - createExpectedRow("2011-04-01", "alias", "n", "rows", 1L, "idx", 121L), - createExpectedRow("2011-04-01", "alias", "p", "rows", 3L, "idx", 2900L), - createExpectedRow("2011-04-01", "alias", "t", "rows", 2L, "idx", 197L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "a", "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "b", "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "e", "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "h", "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "m", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "n", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "p", "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "t", "rows", 2L, "idx", 197L), - createExpectedRow("2011-04-02", "alias", "a", "rows", 1L, "idx", 147L), - createExpectedRow("2011-04-02", "alias", "b", "rows", 1L, "idx", 112L), - createExpectedRow("2011-04-02", "alias", "e", "rows", 1L, "idx", 166L), - createExpectedRow("2011-04-02", "alias", "h", "rows", 1L, "idx", 113L), - createExpectedRow("2011-04-02", "alias", "m", "rows", 3L, "idx", 2447L), - createExpectedRow("2011-04-02", "alias", "n", "rows", 1L, "idx", 114L), - createExpectedRow("2011-04-02", "alias", "p", "rows", 3L, "idx", 2505L), - createExpectedRow("2011-04-02", "alias", "t", "rows", 2L, "idx", 223L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "a", "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "b", "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "e", "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "h", "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "m", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "n", "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "p", "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "t", "rows", 2L, "idx", 223L) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -368,28 +363,172 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "automotive", "rows", 1L, "idx", 135L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "business", "rows", 1L, "idx", 118L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "entertainment", "rows", 1L, "idx", 158L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "health", "rows", 1L, "idx", 120L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "mezzanine", "rows", 3L, "idx", 2870L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "news", "rows", 1L, "idx", 121L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "premium", "rows", 3L, "idx", 2900L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "technology", "rows", 1L, "idx", 78L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "travel", "rows", 1L, "idx", 119L), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "automotive", + "rows", + 1L, + "idx", + 135L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "business", + "rows", + 1L, + "idx", + 118L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "entertainment", + "rows", + 1L, + "idx", + 158L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "health", + "rows", + 1L, + "idx", + 120L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "mezzanine", + "rows", + 3L, + "idx", + 2870L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "news", + "rows", + 1L, + "idx", + 121L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "premium", + "rows", + 3L, + "idx", + 2900L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "technology", + "rows", + 1L, + "idx", + 78L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-03-31", tz), + "alias", + "travel", + "rows", + 1L, + "idx", + 119L + ), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "automotive", "rows", 1L, "idx", 147L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "business", "rows", 1L, "idx", 112L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "entertainment", "rows", 1L, "idx", 166L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "health", "rows", 1L, "idx", 113L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "mezzanine", "rows", 3L, "idx", 2447L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "news", "rows", 1L, "idx", 114L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "premium", "rows", 3L, "idx", 2505L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "technology", "rows", 1L, "idx", 97L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "automotive", + "rows", + 1L, + "idx", + 147L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "business", + "rows", + 1L, + "idx", + 112L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "entertainment", + "rows", + 1L, + "idx", + 166L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "health", + "rows", + 1L, + "idx", + 113L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "mezzanine", + "rows", + 3L, + "idx", + 2447L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "news", + "rows", + 1L, + "idx", + 114L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "premium", + "rows", + 3L, + "idx", + 2505L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "technology", + "rows", + 1L, + "idx", + 97L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + new DateTime("2011-04-01", tz), + "alias", + "travel", + "rows", + 1L, + "idx", + 126L + ) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -416,7 +555,9 @@ public class GroupByQueryRunnerTest new QueryRunner() { @Override - public Sequence run(Query query, Map context) + public Sequence run( + Query query, Map context + ) { // simulate two daily segments final Query query1 = query.withQuerySegmentSpec( @@ -431,31 +572,31 @@ public class GroupByQueryRunnerTest ); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) ); - HashMap context = new HashMap(); + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, runner.run(fullQuery, context), "direct"); TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery, context), "merged"); List allGranExpectedResults = Arrays.asList( - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 2L, "idx", 269L), - createExpectedRow("2011-04-02", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-02", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-02", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L) ); TestHelper.assertExpectedObjects(allGranExpectedResults, runner.run(allGranQuery, context), "direct"); @@ -489,19 +630,20 @@ public class GroupByQueryRunnerTest final GroupByQuery fullQuery = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) ); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); - HashMap context = new HashMap(); + + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery, context), String.format("limit: %d", limit) ); @@ -564,15 +706,15 @@ public class GroupByQueryRunnerTest }; List allResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) ); List> expectedResults = Lists.newArrayList( @@ -609,7 +751,9 @@ public class GroupByQueryRunnerTest new QueryRunner() { @Override - public Sequence run(Query query, Map context) + public Sequence run( + Query query, Map context + ) { // simulate two daily segments final Query query1 = query.withQuerySegmentSpec( @@ -622,7 +766,8 @@ public class GroupByQueryRunnerTest } } ); - HashMap context = new HashMap(); + + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery, context), "merged"); } @@ -647,19 +792,19 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L) ); + Map context = Maps.newHashMap(); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); - HashMap context = new HashMap(); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); TestHelper.assertExpectedObjects( @@ -688,19 +833,19 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L) ); + Map context = Maps.newHashMap(); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); - HashMap context = new HashMap(); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build(), context), "limited" @@ -728,32 +873,164 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4423.6533203125D), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4418.61865234375D), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319.94403076171875D), - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 270.3977966308594D), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243.65843200683594D), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 222.20980834960938D), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 218.7224884033203D), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216.97836303710938D), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 178.24917602539062D) + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "mezzanine", + "rows", + 6L, + "idx", + 4423.6533203125D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "premium", + "rows", + 6L, + "idx", + 4418.61865234375D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 2L, + "idx", + 319.94403076171875D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "automotive", + "rows", + 2L, + "idx", + 270.3977966308594D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "travel", + "rows", + 2L, + "idx", + 243.65843200683594D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "news", + "rows", + 2L, + "idx", + 222.20980834960938D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "business", + "rows", + 2L, + "idx", + 218.7224884033203D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "health", + "rows", + 2L, + "idx", + 216.97836303710938D + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "technology", + "rows", + 2L, + "idx", + 178.24917602539062D + ) ); + Map context = Maps.newHashMap(); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); - HashMap context = new HashMap(); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build(), context), "limited" ); } + @Test + public void testGroupByWithMixedCasingOrdering() + { + GroupByQuery query = new GroupByQuery.Builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setGranularity(QueryRunnerTestHelper.allGran) + .setDimensions( + Arrays.asList( + new DefaultDimensionSpec( + QueryRunnerTestHelper.marketDimension, + "MarketAlias" + ) + ) + ) + .setInterval(QueryRunnerTestHelper.fullOnInterval) + .setLimitSpec( + new DefaultLimitSpec( + Lists.newArrayList( + new OrderByColumnSpec( + "marketALIAS", + OrderByColumnSpec.Direction.DESCENDING + ) + ), 3 + ) + ) + .setAggregatorSpecs( + Lists.newArrayList( + QueryRunnerTestHelper.rowsCount + ) + ) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "marketalias", + "upfront", + "rows", + 186L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "marketalias", + "total_market", + "rows", + 186L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "marketalias", + "spot", + "rows", + 837L + ) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "order-limit"); + } + @Test public void testHavingSpec() { List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L) ); GroupByQuery.Builder builder = GroupByQuery @@ -783,7 +1060,9 @@ public class GroupByQueryRunnerTest new QueryRunner() { @Override - public Sequence run(Query query, Map context) + public Sequence run( + Query query, Map context + ) { // simulate two daily segments final Query query1 = query.withQuerySegmentSpec( @@ -797,7 +1076,7 @@ public class GroupByQueryRunnerTest } ); - HashMap context = new HashMap(); + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery, context), "merged"); } @@ -820,7 +1099,7 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L) ); final GroupByQueryEngine engine = new GroupByQueryEngine( @@ -836,9 +1115,14 @@ public class GroupByQueryRunnerTest } ) ); - HashMap context = new HashMap(); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier,new DefaultObjectMapper(), engine, TestQueryRunners.pool).mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest( + configSupplier, + new DefaultObjectMapper(), + engine, + TestQueryRunners.pool + ).mergeResults(runner); + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); } @@ -861,17 +1145,18 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "business", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "entertainment", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "health", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "mezzanine", "rows", 6L), - createExpectedRow("2011-04-01", "quality", "news", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "premium", "rows", 6L), - createExpectedRow("2011-04-01", "quality", "technology", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "travel", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "business", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "entertainment", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "health", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "mezzanine", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "news", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "premium", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "technology", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "travel", "rows", 2L) ); - HashMap context = new HashMap(); + + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, runner.run(query, context), "normal"); final GroupByQueryEngine engine = new GroupByQueryEngine( configSupplier, @@ -887,9 +1172,13 @@ public class GroupByQueryRunnerTest ) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier, new DefaultObjectMapper(), engine, TestQueryRunners.pool).mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest( + configSupplier, + new DefaultObjectMapper(), + engine, + TestQueryRunners.pool + ).mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); - } @Test @@ -911,18 +1200,18 @@ public class GroupByQueryRunnerTest final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "business", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "entertainment", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "health", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "mezzanine", "rows", 6L), - createExpectedRow("2011-04-01", "quality", "news", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "premium", "rows", 6L), - createExpectedRow("2011-04-01", "quality", "technology", "rows", 2L), - createExpectedRow("2011-04-01", "quality", "travel", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "business", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "entertainment", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "health", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "mezzanine", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "news", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "premium", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "technology", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "travel", "rows", 2L) ); - HashMap context = new HashMap(); + Map context = Maps.newHashMap(); TestHelper.assertExpectedObjects(expectedResults, runner.run(query, context), "normal"); final GroupByQueryEngine engine = new GroupByQueryEngine( configSupplier, @@ -938,7 +1227,12 @@ public class GroupByQueryRunnerTest ) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier, new DefaultObjectMapper(), engine, TestQueryRunners.pool).mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest( + configSupplier, + new DefaultObjectMapper(), + engine, + TestQueryRunners.pool + ).mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query, context), "no-limit"); } @@ -976,29 +1270,29 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) ); // Subqueries are handled by the ToolChest - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1032,11 +1326,11 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "idx", 2900.0), - createExpectedRow("2011-04-02", "idx", 2505.0) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "idx", 2900.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "idx", 2505.0) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1070,10 +1364,10 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-02", "idx", 2505.0) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "idx", 2505.0) ); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1106,7 +1400,7 @@ public class GroupByQueryRunnerTest .setGranularity(QueryRunnerTestHelper.dayGran) .build(); - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); Assert.assertFalse(results.iterator().hasNext()); } @@ -1165,29 +1459,45 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 11135.0), - createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 11118.0), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 11158.0), - createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 11120.0), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 13870.0), - createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 11121.0), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 13900.0), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 11078.0), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 11119.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 11135.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 11118.0), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 1L, + "idx", + 11158.0 + ), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 11120.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 13870.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 11121.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 13900.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 11078.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 11119.0), - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 11147.0), - createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 11112.0), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 11166.0), - createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 11113.0), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 13447.0), - createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 11114.0), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 13505.0), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 11097.0), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 11126.0) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 11147.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 11112.0), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "entertainment", + "rows", + 1L, + "idx", + 11166.0 + ), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 11113.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 13447.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 11114.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 13505.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 11097.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 11126.0) ); // Subqueries are handled by the ToolChest - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1265,27 +1575,43 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 11135.0), - createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 11118.0), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 11158.0), - createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 11120.0), - createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 11121.0), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 11078.0), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 11119.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 11135.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 11118.0), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 1L, + "idx", + 11158.0 + ), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 11120.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 11121.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 11078.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 11119.0), - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 11147.0), - createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 11112.0), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 11166.0), - createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 11113.0), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 13447.0), - createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 11114.0), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 13505.0), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 11097.0), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 11126.0) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 11147.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 11112.0), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "entertainment", + "rows", + 1L, + "idx", + 11166.0 + ), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 11113.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 13447.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 11114.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 13505.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 11097.0), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 11126.0) ); // Subqueries are handled by the ToolChest - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1297,14 +1623,14 @@ public class GroupByQueryRunnerTest .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("provider", "function(dim){ return true; }")) + .setDimFilter(new JavaScriptDimFilter("market", "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, new DoubleSumAggregatorFactory("idx_subagg", "index"), new JavaScriptAggregatorFactory( "js_agg", - Arrays.asList("index", "provider"), + Arrays.asList("index", "market"), "function(current, index, dim){return current + index + dim.length;}", "function(){return 0;}", "function(a,b){return a + b;}" @@ -1382,7 +1708,7 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", "travel", @@ -1393,7 +1719,7 @@ public class GroupByQueryRunnerTest "js_outer_agg", 123.92274475097656 ), - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", "technology", @@ -1404,7 +1730,7 @@ public class GroupByQueryRunnerTest "js_outer_agg", 82.62254333496094 ), - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", "news", @@ -1415,7 +1741,7 @@ public class GroupByQueryRunnerTest "js_outer_agg", 125.58358001708984 ), - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", "health", @@ -1426,7 +1752,7 @@ public class GroupByQueryRunnerTest "js_outer_agg", 124.13470458984375 ), - createExpectedRow( + GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", "entertainment", @@ -1440,7 +1766,7 @@ public class GroupByQueryRunnerTest ); // Subqueries are handled by the ToolChest - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } @@ -1478,51 +1804,109 @@ public class GroupByQueryRunnerTest .build(); List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 282L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 230L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 324L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 233L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 5317L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 235L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 5405L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 175L, "uniq", 1.0002442201269182), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 245L, "uniq", 1.0002442201269182) + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "automotive", + "rows", + 2L, + "idx", + 282L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "business", + "rows", + 2L, + "idx", + 230L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 2L, + "idx", + 324L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "health", + "rows", + 2L, + "idx", + 233L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "mezzanine", + "rows", + 6L, + "idx", + 5317L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "news", + "rows", + 2L, + "idx", + 235L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "premium", + "rows", + 6L, + "idx", + 5405L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "technology", + "rows", + 2L, + "idx", + 175L, + "uniq", + 1.0002442201269182 + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "travel", + "rows", + 2L, + "idx", + 245L, + "uniq", + 1.0002442201269182 + ) ); // Subqueries are handled by the ToolChest - Iterable results = runQuery(query); + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); } - - private Iterable runQuery(GroupByQuery query) - { - - QueryToolChest toolChest = factory.getToolchest(); - QueryRunner theRunner = new FinalizeResultsQueryRunner<>( - toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner)), - toolChest - ); - - HashMap context = new HashMap(); - Sequence queryResult = theRunner.run(query, context); - return Sequences.toList(queryResult, Lists.newArrayList()); - } - - private Row createExpectedRow(final String timestamp, Object... vals) - { - return createExpectedRow(new DateTime(timestamp), vals); - } - - private Row createExpectedRow(final DateTime timestamp, Object... vals) - { - Preconditions.checkArgument(vals.length % 2 == 0); - - Map theVals = Maps.newHashMap(); - for (int i = 0; i < vals.length; i += 2) { - theVals.put(vals[i].toString(), vals[i + 1]); - } - - DateTime ts = new DateTime(timestamp); - return new MapBasedRow(ts, theVals); - } } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java new file mode 100644 index 00000000000..a60335e214c --- /dev/null +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java @@ -0,0 +1,72 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import org.joda.time.DateTime; + +import java.util.Map; + +/** + */ +public class GroupByQueryRunnerTestHelper +{ + public static Iterable runQuery(QueryRunnerFactory factory, QueryRunner runner, GroupByQuery query) + { + + QueryToolChest toolChest = factory.getToolchest(); + QueryRunner theRunner = new FinalizeResultsQueryRunner<>( + toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner)), + toolChest + ); + + Sequence queryResult = theRunner.run(query, Maps.newHashMap()); + return Sequences.toList(queryResult, Lists.newArrayList()); + } + + public static Row createExpectedRow(final String timestamp, Object... vals) + { + return createExpectedRow(new DateTime(timestamp), vals); + } + + public static Row createExpectedRow(final DateTime timestamp, Object... vals) + { + Preconditions.checkArgument(vals.length % 2 == 0); + + Map theVals = Maps.newHashMap(); + for (int i = 0; i < vals.length; i += 2) { + theVals.put(vals[i].toString(), vals[i + 1]); + } + + DateTime ts = new DateTime(timestamp); + return new MapBasedRow(ts, theVals); + } + +} diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 830528ba08b..56842f16ec2 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -88,7 +88,7 @@ public class SearchQueryRunnerTest QueryRunnerTestHelper.qualityDimension, Sets.newHashSet("automotive", "mezzanine", "travel", "health", "entertainment") ); - expectedResults.put(QueryRunnerTestHelper.providerDimension.toLowerCase(), Sets.newHashSet("total_market")); + expectedResults.put(QueryRunnerTestHelper.marketDimension.toLowerCase(), Sets.newHashSet("total_market")); expectedResults.put(QueryRunnerTestHelper.placementishDimension, Sets.newHashSet("a")); checkSearchQuery(searchQuery, expectedResults); @@ -138,13 +138,13 @@ public class SearchQueryRunnerTest public void testSearchWithDimensionProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); + expectedResults.put(QueryRunnerTestHelper.marketDimension, new HashSet(Arrays.asList("total_market"))); checkSearchQuery( Druids.newSearchQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimensions("provider") + .dimensions("market") .intervals(QueryRunnerTestHelper.fullOnInterval) .query("a") .build(), @@ -164,7 +164,7 @@ public class SearchQueryRunnerTest "automotive", "mezzanine", "travel", "health", "entertainment" ) ), - QueryRunnerTestHelper.providerDimension, + QueryRunnerTestHelper.marketDimension, new HashSet( Arrays.asList("total_market") ) @@ -178,7 +178,7 @@ public class SearchQueryRunnerTest .dimensions( Arrays.asList( QueryRunnerTestHelper.qualityDimension, - QueryRunnerTestHelper.providerDimension + QueryRunnerTestHelper.marketDimension ) ) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -192,7 +192,7 @@ public class SearchQueryRunnerTest public void testSearchWithDimensionsPlacementAndProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); + expectedResults.put(QueryRunnerTestHelper.marketDimension, new HashSet(Arrays.asList("total_market"))); checkSearchQuery( Druids.newSearchQueryBuilder() @@ -201,7 +201,7 @@ public class SearchQueryRunnerTest .dimensions( Arrays.asList( QueryRunnerTestHelper.placementishDimension, - QueryRunnerTestHelper.providerDimension + QueryRunnerTestHelper.marketDimension ) ) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -236,15 +236,15 @@ public class SearchQueryRunnerTest public void testSearchWithSingleFilter2() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); + expectedResults.put(QueryRunnerTestHelper.marketDimension, new HashSet(Arrays.asList("total_market"))); checkSearchQuery( Druids.newSearchQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.providerDimension, "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "total_market") .intervals(QueryRunnerTestHelper.fullOnInterval) - .dimensions(QueryRunnerTestHelper.providerDimension) + .dimensions(QueryRunnerTestHelper.marketDimension) .query("a") .build(), expectedResults @@ -261,7 +261,7 @@ public class SearchQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -344,7 +344,7 @@ public class SearchQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("total_market") .build(), Druids.newSelectorDimFilterBuilder() diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 1b324950715..ed000dfa77d 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -66,7 +66,7 @@ public class SelectQueryRunnerTest ); } - private static final String providerLowercase = "provider"; + private static final String providerLowercase = "market"; private final QueryRunner runner; @@ -272,7 +272,7 @@ public class SelectQueryRunnerTest SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), new LegacySegmentSpec(new Interval("2011-01-12/2011-01-14")), - new SelectorDimFilter(QueryRunnerTestHelper.providerDimension, "spot"), + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), QueryRunnerTestHelper.dayGran, Lists.newArrayList(QueryRunnerTestHelper.qualityDimension), Lists.newArrayList(QueryRunnerTestHelper.indexMetric), @@ -369,8 +369,8 @@ public class SelectQueryRunnerTest new LegacySegmentSpec(new Interval("2011-01-12/2011-01-14")), new AndDimFilter( Arrays.asList( - new SelectorDimFilter(QueryRunnerTestHelper.providerDimension, "spot"), - new SelectorDimFilter(QueryRunnerTestHelper.providerDimension, "foo") + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "foo") ) ), QueryRunnerTestHelper.allGran, diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index dc63e292dba..ea0e0e7060d 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -91,7 +91,7 @@ public class TimeseriesQueryRunnerTest QueryGranularity gran = QueryGranularity.DAY; TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .granularity(QueryRunnerTestHelper.dayGran) + .granularity(gran) .intervals(QueryRunnerTestHelper.fullOnInterval) .aggregators( Arrays.asList( @@ -197,7 +197,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "upfront") + .filters(QueryRunnerTestHelper.marketDimension, "upfront") .intervals(QueryRunnerTestHelper.fullOnInterval) .aggregators( Arrays.asList( @@ -209,7 +209,7 @@ public class TimeseriesQueryRunnerTest Assert.assertEquals( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("upfront") .build(), query.getDimensionsFilter() @@ -422,7 +422,7 @@ public class TimeseriesQueryRunnerTest { TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .granularity( new PeriodGranularity( new Period("P7D"), @@ -475,7 +475,7 @@ public class TimeseriesQueryRunnerTest { TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .granularity(QueryGranularity.HOUR) .intervals( Arrays.asList( @@ -543,7 +543,7 @@ public class TimeseriesQueryRunnerTest { TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .granularity( new PeriodGranularity( new Period("PT1H"), @@ -590,7 +590,7 @@ public class TimeseriesQueryRunnerTest { TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .granularity(new PeriodGranularity(new Period("P1M"), null, null)) .intervals( Arrays.asList( @@ -627,7 +627,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query2 = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .granularity("DAY") .intervals( Arrays.asList( @@ -705,7 +705,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "spot", "upfront", "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market") .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Arrays.asList( @@ -755,7 +755,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(new RegexDimFilter(QueryRunnerTestHelper.providerDimension, "^.p.*$")) // spot and upfront + .filters(new RegexDimFilter(QueryRunnerTestHelper.marketDimension, "^.p.*$")) // spot and upfront .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Arrays.asList( @@ -805,7 +805,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "spot") + .filters(QueryRunnerTestHelper.marketDimension, "spot") .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Arrays.asList( @@ -855,7 +855,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "upfront") + .filters(QueryRunnerTestHelper.marketDimension, "upfront") .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Arrays.asList( @@ -905,7 +905,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "total_market") + .filters(QueryRunnerTestHelper.marketDimension, "total_market") .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Arrays.asList( @@ -957,7 +957,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newOrDimFilterBuilder() @@ -1014,7 +1014,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -1072,7 +1072,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -1130,7 +1130,7 @@ public class TimeseriesQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) .filters( - QueryRunnerTestHelper.providerDimension, + QueryRunnerTestHelper.marketDimension, "spot", "upfront", "total_market", @@ -1186,7 +1186,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newOrDimFilterBuilder() @@ -1291,7 +1291,7 @@ public class TimeseriesQueryRunnerTest TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) - .filters(QueryRunnerTestHelper.providerDimension, "billy") + .filters(QueryRunnerTestHelper.marketDimension, "billy") .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) @@ -1336,7 +1336,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("billy") .build(), Druids.newSelectorDimFilterBuilder() @@ -1540,7 +1540,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -1563,7 +1563,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -1602,7 +1602,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newOrDimFilterBuilder() @@ -1624,7 +1624,7 @@ public class TimeseriesQueryRunnerTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .value("spot") .build(), Druids.newOrDimFilterBuilder() diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index fd3522028ef..2e978dd408f 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -105,7 +105,7 @@ public class TopNQueryRunnerTest this.runner = runner; } - private static final String providerDimension = "provider"; + private static final String marketDimension = "market"; @Test public void testFullOnTopN() @@ -113,7 +113,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -137,7 +137,7 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(providerDimension, "total_market") + .put(marketDimension, "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -146,7 +146,7 @@ public class TopNQueryRunnerTest .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(providerDimension, "upfront") + .put(marketDimension, "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -155,7 +155,7 @@ public class TopNQueryRunnerTest .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(providerDimension, "spot") + .put(marketDimension, "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -177,7 +177,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.addRowsIndexConstantMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -201,7 +201,7 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(providerDimension, "total_market") + .put(marketDimension, "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -210,7 +210,7 @@ public class TopNQueryRunnerTest .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(providerDimension, "upfront") + .put(marketDimension, "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -219,7 +219,7 @@ public class TopNQueryRunnerTest .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(providerDimension, "spot") + .put(marketDimension, "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -242,7 +242,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.uniqueMetric) .threshold(3) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -266,7 +266,7 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -275,7 +275,7 @@ public class TopNQueryRunnerTest .put("minIndex", 59.02102279663086D) .build(), ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -284,7 +284,7 @@ public class TopNQueryRunnerTest .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put("provider", "upfront") + .put("market", "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -307,7 +307,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -322,21 +322,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -356,7 +356,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new NumericTopNMetricSpec("uniques")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -371,21 +371,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - "provider", "spot", + "market", "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - "provider", "total_market", + "market", "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - "provider", "upfront", + "market", "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -405,8 +405,8 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(providerDimension, "total_market", "upfront", "spot") - .dimension(providerDimension) + .filters(marketDimension, "total_market", "upfront", "spot") + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -420,21 +420,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -454,8 +454,8 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(providerDimension, "total_market", "upfront") - .dimension(providerDimension) + .filters(marketDimension, "total_market", "upfront") + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -469,14 +469,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -496,8 +496,8 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(providerDimension, "upfront") - .dimension(providerDimension) + .filters(marketDimension, "upfront") + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -511,7 +511,7 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -532,7 +532,7 @@ public class TopNQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -546,21 +546,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 2L, "index", 2591.68359375D, "addRowsIndexConstant", 2594.68359375D, "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 2L, "index", 2508.39599609375D, "addRowsIndexConstant", 2511.39599609375D, "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 2L, "index", 220.63774871826172D, "addRowsIndexConstant", 223.63774871826172D, @@ -581,7 +581,7 @@ public class TopNQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals( @@ -599,21 +599,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 1L, "index", new Float(1447.341160).doubleValue(), "addRowsIndexConstant", new Float(1449.341160).doubleValue(), "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 1L, "index", new Float(1314.839715).doubleValue(), "addRowsIndexConstant", new Float(1316.839715).doubleValue(), "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 1L, "index", new Float(109.705815).doubleValue(), "addRowsIndexConstant", new Float(111.705815).doubleValue(), @@ -633,8 +633,8 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(providerDimension, "total_market", "upfront", "billyblank") - .dimension(providerDimension) + .filters(marketDimension, "total_market", "upfront", "billyblank") + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -648,14 +648,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -675,8 +675,8 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(providerDimension, "billyblank") - .dimension(providerDimension) + .filters(marketDimension, "billyblank") + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -702,7 +702,7 @@ public class TopNQueryRunnerTest .fields( Lists.newArrayList( Druids.newSelectorDimFilterBuilder() - .dimension(providerDimension) + .dimension(marketDimension) .value("billyblank") .build(), Druids.newSelectorDimFilterBuilder() @@ -715,7 +715,7 @@ public class TopNQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(andDimFilter) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -741,7 +741,7 @@ public class TopNQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.placementishDimension, "m") - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -756,7 +756,7 @@ public class TopNQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -958,7 +958,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new LexicographicTopNMetricSpec("")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -972,21 +972,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1006,7 +1006,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new LexicographicTopNMetricSpec("spot")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1020,14 +1020,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1047,7 +1047,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new LexicographicTopNMetricSpec("t")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1061,14 +1061,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1088,7 +1088,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("upfront"))) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1102,14 +1102,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -1129,7 +1129,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("u"))) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1143,14 +1143,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -1172,7 +1172,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("(.)") ) ) .metric("rows") @@ -1188,21 +1188,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "s", + marketDimension, "s", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "t", + marketDimension, "t", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "u", + marketDimension, "u", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1224,7 +1224,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("(.)") ) ) .metric(new LexicographicTopNMetricSpec(null)) @@ -1240,21 +1240,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "s", + marketDimension, "s", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "t", + marketDimension, "t", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "u", + marketDimension, "u", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1276,7 +1276,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("..(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("..(.)") ) ) .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec(null))) @@ -1292,21 +1292,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "t", + marketDimension, "t", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "o", + marketDimension, "o", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "f", + marketDimension, "f", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1328,7 +1328,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("(.)") ) ) .metric(new LexicographicTopNMetricSpec("spot")) @@ -1344,14 +1344,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "t", + marketDimension, "t", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "u", + marketDimension, "u", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1374,7 +1374,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("(.)") ) ) .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("u"))) @@ -1390,14 +1390,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "t", + marketDimension, "t", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "s", + marketDimension, "s", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -1419,7 +1419,7 @@ public class TopNQueryRunnerTest .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - providerDimension, providerDimension, new RegexDimExtractionFn("..(.)") + marketDimension, marketDimension, new RegexDimExtractionFn("..(.)") ) ) .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("p"))) @@ -1435,14 +1435,14 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "o", + marketDimension, "o", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "f", + marketDimension, "f", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1463,7 +1463,7 @@ public class TopNQueryRunnerTest new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(new InvertedTopNMetricSpec(new NumericTopNMetricSpec(QueryRunnerTestHelper.indexMetric))) .threshold(3) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1477,21 +1477,21 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.of( - providerDimension, "spot", + marketDimension, "spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - providerDimension, "upfront", + marketDimension, "upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - providerDimension, "total_market", + marketDimension, "total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, @@ -1511,7 +1511,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -1541,7 +1541,7 @@ public class TopNQueryRunnerTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(providerDimension, "total_market") + .put(marketDimension, "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -1555,7 +1555,7 @@ public class TopNQueryRunnerTest ) .build(), ImmutableMap.builder() - .put(providerDimension, "upfront") + .put(marketDimension, "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -1569,7 +1569,7 @@ public class TopNQueryRunnerTest ) .build(), ImmutableMap.builder() - .put(providerDimension, "spot") + .put(marketDimension, "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -1596,7 +1596,7 @@ public class TopNQueryRunnerTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -1622,7 +1622,7 @@ public class TopNQueryRunnerTest TopNResultValue topNResult = new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "total_market") + .put(QueryRunnerTestHelper.marketDimension, "total_market") .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -1632,7 +1632,7 @@ public class TopNQueryRunnerTest .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "upfront") + .put(QueryRunnerTestHelper.marketDimension, "upfront") .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -1642,7 +1642,7 @@ public class TopNQueryRunnerTest .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "spot") + .put(QueryRunnerTestHelper.marketDimension, "spot") .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java index f2f4ac22f3d..cfe6978e61b 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java @@ -42,7 +42,7 @@ import static io.druid.query.QueryRunnerTestHelper.commonAggregators; import static io.druid.query.QueryRunnerTestHelper.dataSource; import static io.druid.query.QueryRunnerTestHelper.fullOnInterval; import static io.druid.query.QueryRunnerTestHelper.indexMetric; -import static io.druid.query.QueryRunnerTestHelper.providerDimension; +import static io.druid.query.QueryRunnerTestHelper.marketDimension; public class TopNQueryTest { @@ -54,7 +54,7 @@ public class TopNQueryTest Query query = new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(indexMetric) .threshold(4) .intervals(fullOnInterval) diff --git a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java index 35f80552127..6e657bcbeb0 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java @@ -99,7 +99,7 @@ public class TopNUnionQueryTest TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.unionDataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.providerDimension) + .dimension(QueryRunnerTestHelper.marketDimension) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -129,7 +129,7 @@ public class TopNUnionQueryTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "total_market") + .put(QueryRunnerTestHelper.marketDimension, "total_market") .put("rows", 744L) .put("index", 862719.3151855469D) .put("addRowsIndexConstant", 863464.3151855469D) @@ -143,7 +143,7 @@ public class TopNUnionQueryTest ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "upfront") + .put(QueryRunnerTestHelper.marketDimension, "upfront") .put("rows", 744L) .put("index", 768184.4240722656D) .put("addRowsIndexConstant", 768929.4240722656D) @@ -157,7 +157,7 @@ public class TopNUnionQueryTest ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.providerDimension, "spot") + .put(QueryRunnerTestHelper.marketDimension, "spot") .put("rows", 3348L) .put("index", 382426.28929138184D) .put("addRowsIndexConstant", 385775.28929138184D) diff --git a/processing/src/test/java/io/druid/segment/AppendTest.java b/processing/src/test/java/io/druid/segment/AppendTest.java index 017f99c293e..e95c64a415f 100644 --- a/processing/src/test/java/io/druid/segment/AppendTest.java +++ b/processing/src/test/java/io/druid/segment/AppendTest.java @@ -81,7 +81,7 @@ public class AppendTest final QueryGranularity allGran = QueryGranularity.ALL; final String dimensionValue = "dimension"; final String valueValue = "value"; - final String providerDimension = "provider"; + final String marketDimension = "market"; final String qualityDimension = "quality"; final String placementDimension = "placement"; final String placementishDimension = "placementish"; @@ -317,7 +317,7 @@ public class AppendTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 3L) .put("index", 300.0D) .put("addRowsIndexConstant", 304.0D) @@ -327,7 +327,7 @@ public class AppendTest .build(), new HashMap() {{ - put("provider", null); + put("market", null); put("rows", 3L); put("index", 200.0D); put("addRowsIndexConstant", 204.0D); @@ -336,7 +336,7 @@ public class AppendTest put("minIndex", 0.0); }}, ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -364,7 +364,7 @@ public class AppendTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 3L) .put("index", 300.0D) .put("addRowsIndexConstant", 304.0D) @@ -374,7 +374,7 @@ public class AppendTest .build(), new HashMap() {{ - put("provider", null); + put("market", null); put("rows", 3L); put("index", 100.0D); put("addRowsIndexConstant", 104.0D); @@ -383,7 +383,7 @@ public class AppendTest put("minIndex", 0.0); }}, ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -411,7 +411,7 @@ public class AppendTest new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -459,7 +459,7 @@ public class AppendTest new SearchHit(placementishDimension, "a"), new SearchHit(qualityDimension, "automotive"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -481,7 +481,7 @@ public class AppendTest Arrays.asList( new SearchHit(placementishDimension, "a"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -502,7 +502,7 @@ public class AppendTest new SearchResultValue( Arrays.asList( new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -524,7 +524,7 @@ public class AppendTest Arrays.asList( new SearchHit(placementishDimension, "a"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -559,7 +559,7 @@ public class AppendTest .dataSource(dataSource) .granularity(allGran) .intervals(fullOnInterval) - .filters(providerDimension, "breakstuff") + .filters(marketDimension, "breakstuff") .aggregators( Lists.newArrayList( Iterables.concat( @@ -610,11 +610,11 @@ public class AppendTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(providerDimension) + .dimension(marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() - .dimension(providerDimension) + .dimension(marketDimension) .value("total_market") .build() ) @@ -640,7 +640,7 @@ public class AppendTest return new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(indexMetric) .threshold(3) .intervals(fullOnInterval) @@ -664,7 +664,7 @@ public class AppendTest return new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(indexMetric) .threshold(3) .filters( @@ -672,7 +672,7 @@ public class AppendTest .fields( Arrays.asList( Druids.newSelectorDimFilterBuilder() - .dimension(providerDimension) + .dimension(marketDimension) .value("spot") .build(), Druids.newSelectorDimFilterBuilder() @@ -716,7 +716,7 @@ public class AppendTest Druids.newNotDimFilterBuilder() .field( Druids.newSelectorDimFilterBuilder() - .dimension(providerDimension) + .dimension(marketDimension) .value("spot") .build() ).build() diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFull.java b/processing/src/test/java/io/druid/segment/SchemalessTestFull.java index 0efa20822ca..29548d8b55e 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFull.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFull.java @@ -71,7 +71,7 @@ public class SchemalessTestFull final QueryGranularity allGran = QueryGranularity.ALL; final String dimensionValue = "dimension"; final String valueValue = "value"; - final String providerDimension = "provider"; + final String marketDimension = "market"; final String qualityDimension = "quality"; final String placementDimension = "placement"; final String placementishDimension = "placementish"; @@ -133,7 +133,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -142,7 +142,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -161,7 +161,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -182,7 +182,7 @@ public class SchemalessTestFull new SearchHit(placementishDimension, "a"), new SearchHit(qualityDimension, "automotive"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -268,7 +268,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "") + .put("market", "") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -277,7 +277,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -296,7 +296,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "") + .put("market", "") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -305,7 +305,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -407,7 +407,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -417,7 +417,7 @@ public class SchemalessTestFull .build(), new HashMap() {{ - put("provider", null); + put("market", null); put("rows", 1L); put("index", 0.0D); put("addRowsIndexConstant", 2.0D); @@ -436,7 +436,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -540,7 +540,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -549,7 +549,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -568,7 +568,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -587,7 +587,7 @@ public class SchemalessTestFull new SearchResultValue( Arrays.asList( new SearchHit(qualityDimension, "automotive"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -658,7 +658,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -755,7 +755,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -765,7 +765,7 @@ public class SchemalessTestFull .build(), new HashMap() {{ - put("provider", null); + put("market", null); put("rows", 1L); put("index", 0.0D); put("addRowsIndexConstant", 2.0D); @@ -784,7 +784,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -942,7 +942,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 200.0D) .put("addRowsIndexConstant", 202.0D) @@ -1042,7 +1042,7 @@ public class SchemalessTestFull Arrays.>asList( new HashMap() {{ - put("provider", null); + put("market", null); put("rows", 2L); put("index", 200.0D); put("addRowsIndexConstant", 203.0D); @@ -1051,7 +1051,7 @@ public class SchemalessTestFull put("minIndex", 100.0); }}, ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -1070,7 +1070,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -1175,7 +1175,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "") + .put("market", "") .put("rows", 6L) .put("index", 400.0D) .put("addRowsIndexConstant", 407.0D) @@ -1184,7 +1184,7 @@ public class SchemalessTestFull .put("minIndex", 0.0) .build(), ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 4L) .put("index", 400.0D) .put("addRowsIndexConstant", 405.0D) @@ -1193,7 +1193,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -1212,7 +1212,7 @@ public class SchemalessTestFull new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 4L) .put("index", 400.0D) .put("addRowsIndexConstant", 405.0D) @@ -1221,7 +1221,7 @@ public class SchemalessTestFull .put("minIndex", 100.0) .build(), ImmutableMap.builder() - .put("provider", "") + .put("market", "") .put("rows", 1L) .put("index", 100.0D) .put("addRowsIndexConstant", 102.0D) @@ -1242,7 +1242,7 @@ public class SchemalessTestFull new SearchHit(placementishDimension, "a"), new SearchHit(qualityDimension, "automotive"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) @@ -1404,7 +1404,7 @@ public class SchemalessTestFull .dataSource(dataSource) .granularity(allGran) .intervals(fullOnInterval) - .filters(providerDimension, "spot") + .filters(marketDimension, "spot") .aggregators( Lists.newArrayList( Iterables.concat( @@ -1434,7 +1434,7 @@ public class SchemalessTestFull TopNQuery query = new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(indexMetric) .threshold(3) .intervals(fullOnInterval) @@ -1467,8 +1467,8 @@ public class SchemalessTestFull TopNQuery query = new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) - .filters(providerDimension, "spot") + .dimension(marketDimension) + .filters(marketDimension, "spot") .metric(indexMetric) .threshold(3) .intervals(fullOnInterval) @@ -1518,7 +1518,7 @@ public class SchemalessTestFull SearchQuery query = Druids.newSearchQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .filters(providerDimension, "spot") + .filters(marketDimension, "spot") .intervals(fullOnInterval) .query("a") .build(); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java index 153a1226619..9544391c8f8 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimple.java @@ -98,7 +98,7 @@ public class SchemalessTestSimple final QueryGranularity allGran = QueryGranularity.ALL; final String dimensionValue = "dimension"; final String valueValue = "value"; - final String providerDimension = "provider"; + final String marketDimension = "market"; final String qualityDimension = "quality"; final String placementDimension = "placement"; final String placementishDimension = "placementish"; @@ -177,7 +177,7 @@ public class SchemalessTestSimple TopNQuery query = new TopNQueryBuilder() .dataSource(dataSource) .granularity(allGran) - .dimension(providerDimension) + .dimension(marketDimension) .metric(indexMetric) .threshold(3) .intervals(fullOnInterval) @@ -202,7 +202,7 @@ public class SchemalessTestSimple Arrays.asList( new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("provider", "spot") + .put("market", "spot") .put("rows", 4L) .put("index", 400.0D) .put("addRowsIndexConstant", 405.0D) @@ -213,7 +213,7 @@ public class SchemalessTestSimple ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("provider", "") + .put("market", "") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -224,7 +224,7 @@ public class SchemalessTestSimple ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("provider", "total_market") + .put("market", "total_market") .put("rows", 2L) .put("index", 200.0D) .put("addRowsIndexConstant", 203.0D) @@ -261,7 +261,7 @@ public class SchemalessTestSimple new SearchHit(placementishDimension, "a"), new SearchHit(qualityDimension, "automotive"), new SearchHit(placementDimension, "mezzanine"), - new SearchHit(providerDimension, "total_market") + new SearchHit(marketDimension, "total_market") ) ) ) diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 7180a6edb52..466eb1c8ec4 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -56,14 +56,14 @@ public class TestIndex { public static final String[] COLUMNS = new String[]{ "ts", - "provider", + "market", "quALIty", "plAcEmEnT", "pLacementish", "iNdEx", "qualiTy_Uniques" }; - public static final String[] DIMENSIONS = new String[]{"provider", "quALIty", "plAcEmEnT", "pLacementish"}; + public static final String[] DIMENSIONS = new String[]{"market", "quALIty", "plAcEmEnT", "pLacementish"}; public static final String[] METRICS = new String[]{"iNdEx"}; private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); diff --git a/processing/src/test/resources/append.json.1 b/processing/src/test/resources/append.json.1 index aa55a99694a..efce43f58e3 100644 --- a/processing/src/test/resources/append.json.1 +++ b/processing/src/test/resources/append.json.1 @@ -4,7 +4,7 @@ }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "placement":"preferred", "placementish":["a", "preferred"], @@ -12,7 +12,7 @@ }, { "timestamp":"2011-01-14T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "index":100.000000 }, diff --git a/processing/src/test/resources/append.json.2 b/processing/src/test/resources/append.json.2 index fd3f916e9ec..88d7501132d 100644 --- a/processing/src/test/resources/append.json.2 +++ b/processing/src/test/resources/append.json.2 @@ -1,14 +1,14 @@ [ { "timestamp":"2011-01-14T22:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placement":"preferred", "placementish":["h", "preferred"], "index":100.000000 }, { "timestamp":"2011-01-14T23:00:00.000Z", - "provider":"total_market", + "market":"total_market", "quality":"business", "placement":"mezzanine", "placementish":["p", "preferred"], @@ -17,7 +17,7 @@ { "timestamp":"2011-01-15T00:00:00.000Z", "placementish":"preferred", - "provider":"spot", + "market":"spot", "index":100.000000 }, { diff --git a/processing/src/test/resources/append.json.3 b/processing/src/test/resources/append.json.3 index 7a963dc4e78..de15d125969 100644 --- a/processing/src/test/resources/append.json.3 +++ b/processing/src/test/resources/append.json.3 @@ -4,7 +4,7 @@ }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "placement":"preferred", "placementish":["a", "preferred"], @@ -12,7 +12,7 @@ }, { "timestamp":"2011-01-14T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "index":100.000000 }, { diff --git a/processing/src/test/resources/append.json.4 b/processing/src/test/resources/append.json.4 index 5049fdc5e69..1a8d7d5d8d9 100644 --- a/processing/src/test/resources/append.json.4 +++ b/processing/src/test/resources/append.json.4 @@ -1,14 +1,14 @@ [ { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placement":"preferred", "placementish":["h", "preferred"], "index":100.000000 }, { "timestamp":"2011-01-13T01:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placement":"mezzanine", "placementish":["p", "preferred"], "index":100.000000 @@ -16,7 +16,7 @@ { "timestamp":"2011-01-13T15:00:00.000Z", "placementish":"preferred", - "provider":"spot", + "market":"spot", "index":100.000000 }, { diff --git a/processing/src/test/resources/append.json.5 b/processing/src/test/resources/append.json.5 index 0119fead368..a59d23c13c5 100644 --- a/processing/src/test/resources/append.json.5 +++ b/processing/src/test/resources/append.json.5 @@ -1,32 +1,32 @@ [ { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-14T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-16T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-17T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-19T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-21T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 } ] \ No newline at end of file diff --git a/processing/src/test/resources/append.json.6 b/processing/src/test/resources/append.json.6 index 386769ce0ef..56ff8a3faad 100644 --- a/processing/src/test/resources/append.json.6 +++ b/processing/src/test/resources/append.json.6 @@ -1,12 +1,12 @@ [ { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"breakstuff", + "market":"breakstuff", "index":100.000000 }, { "timestamp":"2011-01-15T00:00:00.000Z", - "provider":"spot", + "market":"spot", "index":100.000000 } ] \ No newline at end of file diff --git a/processing/src/test/resources/append.json.7 b/processing/src/test/resources/append.json.7 index 8645df52b8f..cc8089f8981 100644 --- a/processing/src/test/resources/append.json.7 +++ b/processing/src/test/resources/append.json.7 @@ -1,12 +1,12 @@ [ { "timestamp":"2011-01-18T00:00:00.000Z", - "provider":"spot", + "market":"spot", "index":100.000000 }, { "timestamp":"2011-01-20T00:00:00.000Z", - "provider":"spot", + "market":"spot", "index":100.000000 } ] \ No newline at end of file diff --git a/processing/src/test/resources/druid.sample.json b/processing/src/test/resources/druid.sample.json index e188e7be089..a3cddb46ff0 100644 --- a/processing/src/test/resources/druid.sample.json +++ b/processing/src/test/resources/druid.sample.json @@ -4,7 +4,7 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "placement":"preferred", "placementish":["a", "preferred"], @@ -12,7 +12,7 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "index":100.000000 }, @@ -23,14 +23,14 @@ }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placementish":["h", "preferred"], "placement":"preferred", "index":100.000000 }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placementish":["p", "preferred"], "quality":"business", "placement":"mezzanine", @@ -39,7 +39,7 @@ { "timestamp":"2011-01-12T00:00:00.000Z", "placementish":"preferred", - "provider":"spot", + "market":"spot", "index":100.000000 }, { @@ -55,12 +55,12 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"", + "market":"", "index":100.000000 }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":["", "spot"], + "market":["", "spot"], "index":100.000000 } ] \ No newline at end of file diff --git a/processing/src/test/resources/druid.sample.json.bottom b/processing/src/test/resources/druid.sample.json.bottom index a7b3b648a2a..75ea3502c6e 100644 --- a/processing/src/test/resources/druid.sample.json.bottom +++ b/processing/src/test/resources/druid.sample.json.bottom @@ -6,14 +6,14 @@ }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "placement":"preferred", "placementish":["h", "preferred"], "index":100.000000 }, { "timestamp":"2011-01-13T00:00:00.000Z", - "provider":"total_market", + "market":"total_market", "quality":"business", "placement":"mezzanine", "placementish":["p", "preferred"], @@ -22,7 +22,7 @@ { "timestamp":"2011-01-12T00:00:00.000Z", "placementish":"preferred", - "provider":"spot", + "market":"spot", "index":100.000000 }, { @@ -38,12 +38,12 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"", + "market":"", "index":100.000000 }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":["", "spot"], + "market":["", "spot"], "index":100.000000 } ] \ No newline at end of file diff --git a/processing/src/test/resources/druid.sample.json.top b/processing/src/test/resources/druid.sample.json.top index 8439291578d..4d1834ca27a 100644 --- a/processing/src/test/resources/druid.sample.json.top +++ b/processing/src/test/resources/druid.sample.json.top @@ -4,7 +4,7 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "placement":"preferred", "placementish":["a", "preferred"], @@ -12,7 +12,7 @@ }, { "timestamp":"2011-01-12T00:00:00.000Z", - "provider":"spot", + "market":"spot", "quality":"automotive", "index":100.000000 } diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index ff5ce5fa4d1..43cec97c472 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -39,5 +39,11 @@ commons-cli test + + io.druid + druid-processing + ${project.parent.version} + test + diff --git a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java index 132fe3b6179..28f1d0d14d1 100644 --- a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java +++ b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java @@ -19,12 +19,12 @@ package io.druid.firehose.rabbitmq; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Maps; import com.rabbitmq.client.ConnectionFactory; +import com.rabbitmq.client.LongString; -import java.net.URISyntaxException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; import java.util.Map; /** @@ -33,140 +33,229 @@ import java.util.Map; */ public class JacksonifiedConnectionFactory extends ConnectionFactory { + public static JacksonifiedConnectionFactory makeDefaultConnectionFactory() throws Exception + { + return new JacksonifiedConnectionFactory(null, 0, null, null, null, null, 0, 0, 0, 0, null); + } + + private static Map getSerializableClientProperties(final Map clientProperties) + { + return Maps.transformEntries( + clientProperties, + new Maps.EntryTransformer() + { + @Override + public Object transformEntry(String key, Object value) + { + if (value instanceof LongString) { + return value.toString(); + } + return value; + } + } + ); + } + + private final String host; + private final int port; + private final String username; + private final String password; + private final String virtualHost; + private final String uri; + private final int requestedChannelMax; + private final int requestedFrameMax; + private final int requestedHeartbeat; + private final int connectionTimeout; + private final Map clientProperties; + + @JsonCreator + public JacksonifiedConnectionFactory( + @JsonProperty("host") String host, + @JsonProperty("port") int port, + @JsonProperty("username") String username, + @JsonProperty("password") String password, + @JsonProperty("virtualHost") String virtualHost, + @JsonProperty("uri") String uri, + @JsonProperty("requestedChannelMax") int requestedChannelMax, + @JsonProperty("requestedFrameMax") int requestedFrameMax, + @JsonProperty("requestedHeartbeat") int requestedHeartbeat, + @JsonProperty("connectionTimeout") int connectionTimeout, + @JsonProperty("clientProperties") Map clientProperties + ) throws Exception + { + super(); + + this.host = host == null ? super.getHost() : host; + this.port = port == 0 ? super.getPort() : port; + this.username = username == null ? super.getUsername() : username; + this.password = password == null ? super.getPassword() : password; + this.virtualHost = virtualHost == null ? super.getVirtualHost() : virtualHost; + this.uri = uri; + this.requestedChannelMax = requestedChannelMax == 0 ? super.getRequestedChannelMax() : requestedChannelMax; + this.requestedFrameMax = requestedFrameMax == 0 ? super.getRequestedFrameMax() : requestedFrameMax; + this.requestedHeartbeat = requestedHeartbeat == 0 ? super.getRequestedHeartbeat() : requestedHeartbeat; + this.connectionTimeout = connectionTimeout == 0 ? super.getConnectionTimeout() : connectionTimeout; + this.clientProperties = clientProperties == null ? super.getClientProperties() : clientProperties; + + super.setHost(this.host); + super.setPort(this.port); + super.setUsername(this.username); + super.setPassword(this.password); + super.setVirtualHost(this.virtualHost); + if (this.uri != null) { + super.setUri(this.uri); + } + super.setRequestedChannelMax(this.requestedChannelMax); + super.setRequestedFrameMax(this.requestedFrameMax); + super.setRequestedHeartbeat(this.requestedHeartbeat); + super.setConnectionTimeout(this.connectionTimeout); + super.setClientProperties(this.clientProperties); + } + @Override @JsonProperty public String getHost() { - return super.getHost(); - } - - @Override - public void setHost(String host) - { - super.setHost(host); + return host; } @Override @JsonProperty public int getPort() { - return super.getPort(); + return port; } - @Override - public void setPort(int port) - { - super.setPort(port); - } @Override @JsonProperty public String getUsername() { - return super.getUsername(); - } - - @Override - public void setUsername(String username) - { - super.setUsername(username); + return username; } @Override @JsonProperty public String getPassword() { - return super.getPassword(); - } - - @Override - public void setPassword(String password) - { - super.setPassword(password); + return password; } @Override @JsonProperty public String getVirtualHost() { - return super.getVirtualHost(); + return virtualHost; } - @Override - public void setVirtualHost(String virtualHost) - { - super.setVirtualHost(virtualHost); - } - - @Override @JsonProperty - public void setUri(String uriString) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException + public String getUri() { - super.setUri(uriString); + return uri; } @Override @JsonProperty public int getRequestedChannelMax() { - return super.getRequestedChannelMax(); - } - - @Override - public void setRequestedChannelMax(int requestedChannelMax) - { - super.setRequestedChannelMax(requestedChannelMax); + return requestedChannelMax; } @Override @JsonProperty public int getRequestedFrameMax() { - return super.getRequestedFrameMax(); - } - - @Override - public void setRequestedFrameMax(int requestedFrameMax) - { - super.setRequestedFrameMax(requestedFrameMax); + return requestedFrameMax; } @Override @JsonProperty public int getRequestedHeartbeat() { - return super.getRequestedHeartbeat(); - } - - @Override - public void setConnectionTimeout(int connectionTimeout) - { - super.setConnectionTimeout(connectionTimeout); + return requestedHeartbeat; } @Override @JsonProperty public int getConnectionTimeout() { - return super.getConnectionTimeout(); + return connectionTimeout; + } + + @JsonProperty("clientProperties") + public Map getSerializableClientProperties() + { + return getSerializableClientProperties(clientProperties); } @Override - public void setRequestedHeartbeat(int requestedHeartbeat) + public boolean equals(Object o) { - super.setRequestedHeartbeat(requestedHeartbeat); + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + JacksonifiedConnectionFactory that = (JacksonifiedConnectionFactory) o; + + if (connectionTimeout != that.connectionTimeout) { + return false; + } + if (port != that.port) { + return false; + } + if (requestedChannelMax != that.requestedChannelMax) { + return false; + } + if (requestedFrameMax != that.requestedFrameMax) { + return false; + } + if (requestedHeartbeat != that.requestedHeartbeat) { + return false; + } + if (clientProperties != null + ? !Maps.difference( + getSerializableClientProperties(clientProperties), + getSerializableClientProperties(that.clientProperties) + ).areEqual() + : that.clientProperties != null) { + return false; + } + if (host != null ? !host.equals(that.host) : that.host != null) { + return false; + } + if (password != null ? !password.equals(that.password) : that.password != null) { + return false; + } + if (uri != null ? !uri.equals(that.uri) : that.uri != null) { + return false; + } + if (username != null ? !username.equals(that.username) : that.username != null) { + return false; + } + if (virtualHost != null ? !virtualHost.equals(that.virtualHost) : that.virtualHost != null) { + return false; + } + + return true; } @Override - @JsonProperty - public Map getClientProperties() + public int hashCode() { - return super.getClientProperties(); - } - - @Override - public void setClientProperties(Map clientProperties) - { - super.setClientProperties(clientProperties); + int result = host != null ? host.hashCode() : 0; + result = 31 * result + port; + result = 31 * result + (username != null ? username.hashCode() : 0); + result = 31 * result + (password != null ? password.hashCode() : 0); + result = 31 * result + (virtualHost != null ? virtualHost.hashCode() : 0); + result = 31 * result + (uri != null ? uri.hashCode() : 0); + result = 31 * result + requestedChannelMax; + result = 31 * result + requestedFrameMax; + result = 31 * result + requestedHeartbeat; + result = 31 * result + connectionTimeout; + result = 31 * result + (clientProperties != null ? clientProperties.hashCode() : 0); + return result; } } diff --git a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java index 548cbcc1d1a..8ca79bbea99 100644 --- a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java +++ b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java @@ -29,7 +29,7 @@ import java.util.List; /** */ -public class RabbitMQDruidModule implements DruidModule +public class RabbitMQDruidModule implements DruidModule { @Override public List getJacksonModules() diff --git a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java index 7bae291c8a3..325bc41cd92 100644 --- a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java +++ b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java @@ -19,6 +19,7 @@ package io.druid.firehose.rabbitmq; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; /** @@ -26,17 +27,50 @@ import com.fasterxml.jackson.annotation.JsonProperty; */ public class RabbitMQFirehoseConfig { - private String queue = null; - private String exchange = null; - private String routingKey = null; - private boolean durable = false; - private boolean exclusive = false; - private boolean autoDelete = false; - // Lyra (auto reconnect) properties - private int maxRetries = 100; - private int retryIntervalSeconds = 2; - private long maxDurationSeconds = 5 * 60; + private static final int defaultMaxRetries = 100; + private static final int defaultRetryIntervalSeconds = 2; + private static final long defaultMaxDurationSeconds = 5 * 60; + + public static RabbitMQFirehoseConfig makeDefaultConfig() + { + return new RabbitMQFirehoseConfig(null, null, null, false, false, false, 0, 0, 0); + } + + private final String queue; + private final String exchange; + private final String routingKey; + private final boolean durable; + private final boolean exclusive; + private final boolean autoDelete; + private final int maxRetries; + private final int retryIntervalSeconds; + private final long maxDurationSeconds; + + @JsonCreator + public RabbitMQFirehoseConfig( + @JsonProperty("queue") String queue, + @JsonProperty("exchange") String exchange, + @JsonProperty("routingKey") String routingKey, + @JsonProperty("durable") boolean durable, + @JsonProperty("exclusive") boolean exclusive, + @JsonProperty("autoDelete") boolean autoDelete, + @JsonProperty("maxRetries") int maxRetries, + @JsonProperty("retryIntervalSeconds") int retryIntervalSeconds, + @JsonProperty("maxDurationSeconds") long maxDurationSeconds + ) + { + this.queue = queue; + this.exchange = exchange; + this.routingKey = routingKey; + this.durable = durable; + this.exclusive = exclusive; + this.autoDelete = autoDelete; + + this.maxRetries = maxRetries == 0 ? defaultMaxRetries : maxRetries; + this.retryIntervalSeconds = retryIntervalSeconds == 0 ? defaultRetryIntervalSeconds : retryIntervalSeconds; + this.maxDurationSeconds = maxDurationSeconds == 0 ? defaultMaxDurationSeconds : maxDurationSeconds; + } @JsonProperty public String getQueue() @@ -44,90 +78,109 @@ public class RabbitMQFirehoseConfig return queue; } - public void setQueue(String queue) - { - this.queue = queue; - } - @JsonProperty public String getExchange() { return exchange; } - public void setExchange(String exchange) - { - this.exchange = exchange; - } - @JsonProperty public String getRoutingKey() { return routingKey; } - public void setRoutingKey(String routingKey) - { - this.routingKey = routingKey; - } - @JsonProperty public boolean isDurable() { return durable; } - public void setDurable(boolean durable) - { - this.durable = durable; - } - @JsonProperty public boolean isExclusive() { return exclusive; } - public void setExclusive(boolean exclusive) - { - this.exclusive = exclusive; - } - @JsonProperty public boolean isAutoDelete() { return autoDelete; } - public void setAutoDelete(boolean autoDelete) - { - this.autoDelete = autoDelete; - } - @JsonProperty - public int getMaxRetries() { + public int getMaxRetries() + { return maxRetries; } - public void setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - } - @JsonProperty - public int getRetryIntervalSeconds() { + public int getRetryIntervalSeconds() + { return retryIntervalSeconds; } - public void setRetryIntervalSeconds(int retryIntervalSeconds) { - this.retryIntervalSeconds = retryIntervalSeconds; - } - @JsonProperty - public long getMaxDurationSeconds() { + public long getMaxDurationSeconds() + { return maxDurationSeconds; } - public void setMaxDurationSeconds(int maxDurationSeconds) { - this.maxDurationSeconds = maxDurationSeconds; + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + RabbitMQFirehoseConfig that = (RabbitMQFirehoseConfig) o; + + if (autoDelete != that.autoDelete) { + return false; + } + if (durable != that.durable) { + return false; + } + if (exclusive != that.exclusive) { + return false; + } + if (maxDurationSeconds != that.maxDurationSeconds) { + return false; + } + if (maxRetries != that.maxRetries) { + return false; + } + if (retryIntervalSeconds != that.retryIntervalSeconds) { + return false; + } + if (exchange != null ? !exchange.equals(that.exchange) : that.exchange != null) { + return false; + } + if (queue != null ? !queue.equals(that.queue) : that.queue != null) { + return false; + } + if (routingKey != null ? !routingKey.equals(that.routingKey) : that.routingKey != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = queue != null ? queue.hashCode() : 0; + result = 31 * result + (exchange != null ? exchange.hashCode() : 0); + result = 31 * result + (routingKey != null ? routingKey.hashCode() : 0); + result = 31 * result + (durable ? 1 : 0); + result = 31 * result + (exclusive ? 1 : 0); + result = 31 * result + (autoDelete ? 1 : 0); + result = 31 * result + maxRetries; + result = 31 * result + retryIntervalSeconds; + result = 31 * result + (int) (maxDurationSeconds ^ (maxDurationSeconds >>> 32)); + return result; } } diff --git a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java index 8504c70ed9d..f38398bcc84 100644 --- a/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java +++ b/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java @@ -26,7 +26,6 @@ import com.metamx.common.logger.Logger; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.client.ConsumerCancelledException; import com.rabbitmq.client.DefaultConsumer; import com.rabbitmq.client.Envelope; @@ -50,14 +49,14 @@ import java.util.concurrent.LinkedBlockingQueue; /** * A FirehoseFactory for RabbitMQ. - * + *

* It will receive it's configuration through the realtime.spec file and expects to find a * consumerProps element in the firehose definition with values for a number of configuration options. * Below is a complete example for a RabbitMQ firehose configuration with some explanation. Options * that have defaults can be skipped but options with no defaults must be specified with the exception * of the URI property. If the URI property is set, it will override any other property that was also * set. - * + *

* File: realtime.spec * 

  *   "firehose" : {
@@ -89,7 +88,7 @@ import java.util.concurrent.LinkedBlockingQueue;
  *     }
  *   },
  *
- * + *

* Limitations: This implementation will not attempt to reconnect to the MQ broker if the * connection to it is lost. Furthermore it does not support any automatic failover on high availability * RabbitMQ clusters. This is not supported by the underlying AMQP client library and while the behavior @@ -97,7 +96,7 @@ import java.util.concurrent.LinkedBlockingQueue; * the RabbitMQ cluster that sets the "ha-mode" and "ha-sync-mode" properly on the queue that this * Firehose connects to, messages should survive an MQ broker node failure and be delivered once a * connection to another node is set up. - * + *

* For more information on RabbitMQ high availability please see: * http://www.rabbitmq.com/ha.html. */ @@ -105,27 +104,36 @@ public class RabbitMQFirehoseFactory implements FirehoseFactory _queue; - public QueueingConsumer(Channel ch) { + public QueueingConsumer(Channel ch) + { this(ch, new LinkedBlockingQueue()); } - public QueueingConsumer(Channel ch, BlockingQueue q) { + public QueueingConsumer(Channel ch, BlockingQueue q) + { super(ch); this._queue = q; } - @Override public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) { + @Override + public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) + { _queue.clear(); } - @Override public void handleCancel(String consumerTag) throws IOException { + @Override + public void handleCancel(String consumerTag) throws IOException + { _queue.clear(); } - @Override public void handleDelivery(String consumerTag, - Envelope envelope, - AMQP.BasicProperties properties, - byte[] body) - throws IOException + @Override + public void handleDelivery( + String consumerTag, + Envelope envelope, + AMQP.BasicProperties properties, + byte[] body + ) + throws IOException { this._queue.add(new Delivery(envelope, properties, body)); } public Delivery nextDelivery() - throws InterruptedException, ShutdownSignalException, ConsumerCancelledException + throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { return _queue.take(); } diff --git a/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java b/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java new file mode 100644 index 00000000000..eca04717fc2 --- /dev/null +++ b/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java @@ -0,0 +1,139 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.examples.rabbitmq; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.rabbitmq.client.ConnectionFactory; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.JSONParseSpec; +import io.druid.data.input.impl.SpatialDimensionSchema; +import io.druid.data.input.impl.StringInputRowParser; +import io.druid.data.input.impl.TimestampSpec; +import io.druid.firehose.rabbitmq.JacksonifiedConnectionFactory; +import io.druid.firehose.rabbitmq.RabbitMQFirehoseConfig; +import io.druid.firehose.rabbitmq.RabbitMQFirehoseFactory; +import io.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + */ +public class RabbitMQFirehoseFactoryTest +{ + private static final ObjectMapper mapper = new DefaultObjectMapper(); + + @Test + public void testSerde() throws Exception + { + RabbitMQFirehoseConfig config = new RabbitMQFirehoseConfig( + "test", + "test2", + "test3", + true, + true, + true, + 5, + 10, + 20 + ); + + JacksonifiedConnectionFactory connectionFactory = new JacksonifiedConnectionFactory( + "foo", + 9978, + "user", + "pw", + "host", + null, + 5, + 10, + 11, + 12, + ImmutableMap.of("hi", "bye") + ); + + RabbitMQFirehoseFactory factory = new RabbitMQFirehoseFactory( + connectionFactory, + config, + new StringInputRowParser( + new JSONParseSpec( + new TimestampSpec("timestamp", "auto"), + new DimensionsSpec( + Arrays.asList("dim"), + Lists.newArrayList(), + Lists.newArrayList() + ) + ), + null, null, null, null + ) + ); + + byte[] bytes = mapper.writeValueAsBytes(factory); + RabbitMQFirehoseFactory factory2 = mapper.readValue(bytes, RabbitMQFirehoseFactory.class); + byte[] bytes2 = mapper.writeValueAsBytes(factory2); + + Assert.assertArrayEquals(bytes, bytes2); + + Assert.assertEquals(factory.getConfig(), factory2.getConfig()); + Assert.assertEquals(factory.getConnectionFactory(), factory2.getConnectionFactory()); + } + + @Test + public void testDefaultSerde() throws Exception + { + RabbitMQFirehoseConfig config = RabbitMQFirehoseConfig.makeDefaultConfig(); + + JacksonifiedConnectionFactory connectionFactory = JacksonifiedConnectionFactory.makeDefaultConnectionFactory(); + + RabbitMQFirehoseFactory factory = new RabbitMQFirehoseFactory( + connectionFactory, + config, + new StringInputRowParser( + new JSONParseSpec( + new TimestampSpec("timestamp", "auto"), + new DimensionsSpec( + Arrays.asList("dim"), + Lists.newArrayList(), + Lists.newArrayList() + ) + ), + null, null, null, null + ) + ); + + byte[] bytes = mapper.writeValueAsBytes(factory); + RabbitMQFirehoseFactory factory2 = mapper.readValue(bytes, RabbitMQFirehoseFactory.class); + byte[] bytes2 = mapper.writeValueAsBytes(factory2); + + Assert.assertArrayEquals(bytes, bytes2); + + Assert.assertEquals(factory.getConfig(), factory2.getConfig()); + Assert.assertEquals(factory.getConnectionFactory(), factory2.getConnectionFactory()); + + Assert.assertEquals(300, factory2.getConfig().getMaxDurationSeconds()); + + Assert.assertEquals(ConnectionFactory.DEFAULT_HOST, factory2.getConnectionFactory().getHost()); + Assert.assertEquals(ConnectionFactory.DEFAULT_USER, factory2.getConnectionFactory().getUsername()); + Assert.assertEquals(ConnectionFactory.DEFAULT_AMQP_PORT, factory2.getConnectionFactory().getPort()); + } +} diff --git a/server/pom.xml b/server/pom.xml index 0b9726a61b5..aa0b1508119 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -18,7 +18,8 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid druid-server @@ -77,6 +78,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-smile-provider + com.fasterxml.jackson.dataformat jackson-dataformat-smile diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index ba3b49f25d2..10af6f0d8e7 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -25,7 +25,6 @@ import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -261,14 +260,22 @@ public class CachingClusteredClient implements QueryRunner Ordering.natural().onResultOf(Pair.>lhsFn()) ); - final Sequence> seq = Sequences.simple( - Iterables.transform(listOfSequences, Pair.>rhsFn()) - ); - if (strategy == null) { - return toolChest.mergeSequences(seq); - } else { - return strategy.mergeSequences(seq); + final List> orderedSequences = Lists.newLinkedList(); + DateTime unorderedStart = null; + List> unordered = Lists.newLinkedList(); + for (Pair> sequencePair : listOfSequences) { + if (unorderedStart != null && unorderedStart.getMillis() != sequencePair.lhs.getMillis()) { + orderedSequences.add(toolChest.mergeSequencesUnordered(Sequences.simple(unordered))); + unordered = Lists.newLinkedList(); + } + unorderedStart = sequencePair.lhs; + unordered.add(sequencePair.rhs); } + if(!unordered.isEmpty()) { + orderedSequences.add(toolChest.mergeSequencesUnordered(Sequences.simple(unordered))); + } + + return toolChest.mergeSequences(Sequences.simple(orderedSequences)); } private void addSequencesFromCache(ArrayList>> listOfSequences) @@ -332,7 +339,9 @@ public class CachingClusteredClient implements QueryRunner if (!server.isAssignable() || !populateCache || isBySegment) { resultSeqToAdd = clientQueryable.run(query.withQuerySegmentSpec(segmentSpec), context); } else { - resultSeqToAdd = toolChest.mergeSequences( + // this could be more efficient, since we only need to reorder results + // for batches of segments with the same segment start time. + resultSeqToAdd = toolChest.mergeSequencesUnordered( Sequences.map( clientQueryable.run(rewrittenQuery.withQuerySegmentSpec(segmentSpec), context), new Function>() diff --git a/server/src/main/java/io/druid/server/coordination/BaseZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/BaseZkCoordinator.java index f31cfb6311c..43776480829 100644 --- a/server/src/main/java/io/druid/server/coordination/BaseZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/BaseZkCoordinator.java @@ -37,7 +37,6 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.utils.ZKPaths; import java.io.IOException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** @@ -212,6 +211,11 @@ public abstract class BaseZkCoordinator implements DataSegmentChangeHandler } } + public boolean isStarted() + { + return started; + } + public abstract void loadLocalCache(); public abstract DataSegmentChangeHandler getDataSegmentChangeHandler(); diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 34a0ade4574..ba139e8a370 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -169,7 +169,7 @@ public class ZkCoordinator extends BaseZkCoordinator catch (IOException e) { throw new SegmentLoadingException(e, "Failed to announce segment[%s]", segment.getIdentifier()); } - }; + } } catch (SegmentLoadingException e) { log.makeAlert(e, "Failed to load segment for dataSource") diff --git a/server/src/main/java/io/druid/server/http/HistoricalResource.java b/server/src/main/java/io/druid/server/http/HistoricalResource.java new file mode 100644 index 00000000000..a4f5956a0cc --- /dev/null +++ b/server/src/main/java/io/druid/server/http/HistoricalResource.java @@ -0,0 +1,32 @@ +package io.druid.server.http; + +import com.google.common.collect.ImmutableMap; +import io.druid.server.coordination.ZkCoordinator; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +@Path("/druid/historical/v1") +public class HistoricalResource +{ + private final ZkCoordinator coordinator; + + @Inject + public HistoricalResource( + ZkCoordinator coordinator + ) + { + this.coordinator = coordinator; + } + + @GET + @Path("/loadstatus") + @Produces("application/json") + public Response getLoadStatus() + { + return Response.ok(ImmutableMap.of("cacheInitialized", coordinator.isStarted())).build(); + } +} diff --git a/server/src/main/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategy.java new file mode 100644 index 00000000000..afc7997de43 --- /dev/null +++ b/server/src/main/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategy.java @@ -0,0 +1,106 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import io.druid.query.Query; + +import javax.script.Compilable; +import javax.script.Invocable; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + +public class JavaScriptTieredBrokerSelectorStrategy implements TieredBrokerSelectorStrategy +{ + public static interface SelectorFunction + { + public String apply(TieredBrokerConfig config, Query query); + } + + private final SelectorFunction fnSelector; + private final String function; + + @JsonCreator + public JavaScriptTieredBrokerSelectorStrategy(@JsonProperty("function") String fn) + { + Preconditions.checkNotNull(fn, "function must not be null"); + + final ScriptEngine engine = new ScriptEngineManager().getEngineByName("javascript"); + try { + ((Compilable)engine).compile("var apply = " + fn).eval(); + } catch(ScriptException e) { + Throwables.propagate(e); + } + this.function = fn; + this.fnSelector = ((Invocable)engine).getInterface(SelectorFunction.class); + } + + @Override + public Optional getBrokerServiceName( + TieredBrokerConfig config, Query query + ) + { + return Optional.fromNullable(fnSelector.apply(config, query)); + } + + @JsonProperty + public String getFunction() + { + return function; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + JavaScriptTieredBrokerSelectorStrategy that = (JavaScriptTieredBrokerSelectorStrategy) o; + + if (!function.equals(that.function)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return function.hashCode(); + } + + @Override + public String toString() + { + return "JavascriptTieredBrokerSelectorStrategy{" + + "function='" + function + '\'' + + '}'; + } +} diff --git a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java index 40a7714d870..10bffe4fb37 100644 --- a/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java +++ b/server/src/main/java/io/druid/server/router/TieredBrokerSelectorStrategy.java @@ -29,7 +29,8 @@ import io.druid.query.Query; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "timeBoundary", value = TimeBoundaryTieredBrokerSelectorStrategy.class), - @JsonSubTypes.Type(name = "priority", value = PriorityTieredBrokerSelectorStrategy.class) + @JsonSubTypes.Type(name = "priority", value = PriorityTieredBrokerSelectorStrategy.class), + @JsonSubTypes.Type(name = "javascript", value = JavaScriptTieredBrokerSelectorStrategy.class) }) public interface TieredBrokerSelectorStrategy diff --git a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java index be640a03545..56ff5159c62 100644 --- a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java @@ -49,14 +49,14 @@ public class HashBasedNumberedShardSpec extends NumberedShardSpec } @Override - public boolean isInChunk(InputRow inputRow) + public boolean isInChunk(long timestamp, InputRow inputRow) { - return (((long) hash(inputRow)) - getPartitionNum()) % getPartitions() == 0; + return (((long) hash(timestamp, inputRow)) - getPartitionNum()) % getPartitions() == 0; } - protected int hash(InputRow inputRow) + protected int hash(long timestamp, InputRow inputRow) { - final List groupKey = Rows.toGroupKey(inputRow.getTimestampFromEpoch(), inputRow); + final List groupKey = Rows.toGroupKey(timestamp, inputRow); try { return hashFunction.hashBytes(jsonMapper.writeValueAsBytes(groupKey)).asInt(); } @@ -80,9 +80,9 @@ public class HashBasedNumberedShardSpec extends NumberedShardSpec return new ShardSpecLookup() { @Override - public ShardSpec getShardSpec(InputRow row) + public ShardSpec getShardSpec(long timestamp, InputRow row) { - int index = Math.abs(hash(row) % getPartitions()); + int index = Math.abs(hash(timestamp, row) % getPartitions()); return shardSpecs.get(index); } }; diff --git a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java index d3ab608a328..b095f7135c4 100644 --- a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java @@ -50,7 +50,7 @@ public class LinearShardSpec implements ShardSpec return new ShardSpecLookup() { @Override - public ShardSpec getShardSpec(InputRow row) + public ShardSpec getShardSpec(long timestamp, InputRow row) { return shardSpecs.get(0); } @@ -63,7 +63,7 @@ public class LinearShardSpec implements ShardSpec } @Override - public boolean isInChunk(InputRow inputRow) { + public boolean isInChunk(long timestamp, InputRow inputRow) { return true; } diff --git a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java index 65399f1009f..7c90e66c711 100644 --- a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java @@ -60,7 +60,7 @@ public class NumberedShardSpec implements ShardSpec return new ShardSpecLookup() { @Override - public ShardSpec getShardSpec(InputRow row) + public ShardSpec getShardSpec(long timestamp, InputRow row) { return shardSpecs.get(0); } @@ -80,7 +80,7 @@ public class NumberedShardSpec implements ShardSpec } @Override - public boolean isInChunk(InputRow inputRow) + public boolean isInChunk(long timestamp, InputRow inputRow) { return true; } diff --git a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index 1cf232a4b29..2350dc559ef 100644 --- a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -100,10 +100,10 @@ public class SingleDimensionShardSpec implements ShardSpec return new ShardSpecLookup() { @Override - public ShardSpec getShardSpec(InputRow row) + public ShardSpec getShardSpec(long timestamp, InputRow row) { for (ShardSpec spec : shardSpecs) { - if (spec.isInChunk(row)) { + if (spec.isInChunk(timestamp, row)) { return spec; } } @@ -124,7 +124,7 @@ public class SingleDimensionShardSpec implements ShardSpec } @Override - public boolean isInChunk(InputRow inputRow) + public boolean isInChunk(long timestamp, InputRow inputRow) { final List values = inputRow.getDimension(dimension); diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 212e80f46bf..17abc5c3214 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -309,6 +309,64 @@ public class CachingClusteredClientTest ); } + @Test + public void testTimeseriesMergingOutOfOrderPartitions() throws Exception + { + final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals(SEG_SPEC) + .filters(DIM_FILTER) + .granularity(GRANULARITY) + .aggregators(AGGS) + .postAggregators(POST_AGGS) + .context(CONTEXT); + + QueryRunner runner = new FinalizeResultsQueryRunner(client, new TimeseriesQueryQueryToolChest(new QueryConfig())); + + testQueryCaching( + runner, + builder.build(), + new Interval("2011-01-05/2011-01-10"), + makeTimeResults( + new DateTime("2011-01-05T02"), 80, 100, + new DateTime("2011-01-06T02"), 420, 520, + new DateTime("2011-01-07T02"), 12, 2194, + new DateTime("2011-01-08T02"), 59, 201, + new DateTime("2011-01-09T02"), 181, 52 + ), + new Interval("2011-01-05/2011-01-10"), + makeTimeResults( + new DateTime("2011-01-05T00"), 85, 102, + new DateTime("2011-01-06T00"), 412, 521, + new DateTime("2011-01-07T00"), 122, 21894, + new DateTime("2011-01-08T00"), 5, 20, + new DateTime("2011-01-09T00"), 18, 521 + ) + ); + + TestHelper.assertExpectedResults( + makeRenamedTimeResults( + new DateTime("2011-01-05T00"), 85, 102, + new DateTime("2011-01-05T02"), 80, 100, + new DateTime("2011-01-06T00"), 412, 521, + new DateTime("2011-01-06T02"), 420, 520, + new DateTime("2011-01-07T00"), 122, 21894, + new DateTime("2011-01-07T02"), 12, 2194, + new DateTime("2011-01-08T00"), 5, 20, + new DateTime("2011-01-08T02"), 59, 201, + new DateTime("2011-01-09T00"), 18, 521, + new DateTime("2011-01-09T02"), 181, 52 + ), + runner.run( + builder.intervals("2011-01-05/2011-01-10") + .aggregators(RENAMED_AGGS) + .postAggregators(RENAMED_POST_AGGS) + .build(), + Maps.newHashMap() + ) + ); + } + @Test @SuppressWarnings("unchecked") public void testTimeseriesCachingTimeZone() throws Exception diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index c037d5b2b13..166638785f3 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -123,7 +123,10 @@ public class RealtimePlumberSchoolTest @Override public ParseSpec getParseSpec() { - return new JSONParseSpec(new TimestampSpec("timestamp", "auto"), new DimensionsSpec(null, null, null)); + return new JSONParseSpec( + new TimestampSpec("timestamp", "auto"), + new DimensionsSpec(null, null, null) + ); } @Override diff --git a/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java b/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java new file mode 100644 index 00000000000..2b2162148ed --- /dev/null +++ b/server/src/test/java/io/druid/server/router/JavaScriptTieredBrokerSelectorStrategyTest.java @@ -0,0 +1,117 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedHashMap; + +public class JavaScriptTieredBrokerSelectorStrategyTest +{ + final TieredBrokerSelectorStrategy jsStrategy = new JavaScriptTieredBrokerSelectorStrategy( + "function (config, query) { if (config.getTierToBrokerMap().values().size() > 0 && query.getAggregatorSpecs && query.getAggregatorSpecs().size() <= 2) { return config.getTierToBrokerMap().values().toArray()[0] } else { return config.getDefaultBrokerServiceName() } }" + ); + + @Test + public void testSerde() throws Exception + { + ObjectMapper mapper = new DefaultObjectMapper(); + Assert.assertEquals( + jsStrategy, + mapper.readValue( + mapper.writeValueAsString(jsStrategy), + JavaScriptTieredBrokerSelectorStrategy.class + ) + ); + } + + @Test + public void testGetBrokerServiceName() throws Exception + { + final LinkedHashMap tierBrokerMap = new LinkedHashMap<>(); + tierBrokerMap.put("fast", "druid/fastBroker"); + tierBrokerMap.put("slow", "druid/broker"); + + final TieredBrokerConfig tieredBrokerConfig = new TieredBrokerConfig() + { + @Override + public String getDefaultBrokerServiceName() + { + return "druid/broker"; + } + + @Override + public LinkedHashMap getTierToBrokerMap() + { + return tierBrokerMap; + } + }; + + final Druids.TimeseriesQueryBuilder queryBuilder = Druids.newTimeseriesQueryBuilder().dataSource("test") + .intervals("2014/2015") + .aggregators( + ImmutableList.of( + new CountAggregatorFactory("count") + ) + ); + + Assert.assertEquals( + Optional.of("druid/fastBroker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + queryBuilder.build() + ) + ); + + + Assert.assertEquals( + Optional.of("druid/broker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + Druids.newTimeBoundaryQueryBuilder().dataSource("test").bound("maxTime").build() + ) + ); + + Assert.assertEquals( + Optional.of("druid/broker"), + jsStrategy.getBrokerServiceName( + tieredBrokerConfig, + queryBuilder.aggregators( + ImmutableList.of( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("longSum", "a"), + new DoubleSumAggregatorFactory("doubleSum", "b") + ) + ).build() + ) + ); + + } +} diff --git a/server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java b/server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java index bfe5c2df4dd..e353fe0eced 100644 --- a/server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/HashBasedNumberedShardSpecTest.java @@ -127,7 +127,7 @@ public class HashBasedNumberedShardSpecTest public boolean assertExistsInOneSpec(List specs, InputRow row) { for (ShardSpec spec : specs) { - if (spec.isInChunk(row)) { + if (spec.isInChunk(row.getTimestampFromEpoch(), row)) { return true; } } @@ -145,7 +145,7 @@ public class HashBasedNumberedShardSpecTest } @Override - protected int hash(InputRow inputRow) + protected int hash(long timestamp, InputRow inputRow) { return inputRow.hashCode(); } @@ -214,4 +214,5 @@ public class HashBasedNumberedShardSpecTest return 0; } } + } diff --git a/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java index d07b52d9fcb..5bb49b38e5c 100644 --- a/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java @@ -111,7 +111,7 @@ public class SingleDimensionShardSpecTest } ) ); - Assert.assertEquals(String.format("spec[%s], row[%s]", spec, inputRow), pair.lhs, spec.isInChunk(inputRow)); + Assert.assertEquals(String.format("spec[%s], row[%s]", spec, inputRow), pair.lhs, spec.isInChunk(inputRow.getTimestampFromEpoch(), inputRow)); } } } diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index b1098114e72..c34b91d2188 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -39,6 +39,7 @@ import io.druid.query.QuerySegmentWalker; import io.druid.server.QueryResource; import io.druid.server.coordination.ServerManager; import io.druid.server.coordination.ZkCoordinator; +import io.druid.server.http.HistoricalResource; import io.druid.server.initialization.JettyServerInitializer; import io.druid.server.metrics.MetricsModule; import org.eclipse.jetty.server.Server; @@ -72,6 +73,8 @@ public class CliHistorical extends ServerRunnable binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/historical"); binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8083); + // register Server before binding ZkCoordinator to ensure HTTP endpoints are available immediately + LifecycleModule.register(binder, Server.class); binder.bind(ServerManager.class).in(LazySingleton.class); binder.bind(ZkCoordinator.class).in(ManageLifecycle.class); binder.bind(QuerySegmentWalker.class).to(ServerManager.class).in(LazySingleton.class); @@ -79,10 +82,10 @@ public class CliHistorical extends ServerRunnable binder.bind(NodeTypeConfig.class).toInstance(new NodeTypeConfig("historical")); binder.bind(JettyServerInitializer.class).to(QueryJettyServerInitializer.class).in(LazySingleton.class); Jerseys.addResource(binder, QueryResource.class); + Jerseys.addResource(binder, HistoricalResource.class); LifecycleModule.register(binder, QueryResource.class); - LifecycleModule.register(binder, ZkCoordinator.class); - LifecycleModule.register(binder, Server.class); + LifecycleModule.register(binder, ZkCoordinator.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class);