diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 795430b3d7a..5682138b49d 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 8b657d43b7b..da437e99e1f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/docs/content/Best-Practices.md b/docs/content/Best-Practices.md new file mode 100644 index 00000000000..f4cd8f32595 --- /dev/null +++ b/docs/content/Best-Practices.md @@ -0,0 +1,23 @@ +--- +layout: doc_page +--- + +Best Practices +============== + +# Use UTC Timezone + +We recommend using UTC timezone for all your events and across on your nodes, not just for Druid, but for all data infrastructure. This can greatly mitigate potential query problems with inconsistent timezones. + +# Use Lowercase Strings for Column Names + +Druid is not perfect in how it handles mix-cased dimension and metric names. This will hopefully change very soon but for the time being, lower casing your column names is recommended. + +# SSDs + +SSDs are highly recommended for historical and real-time nodes if you are not running a cluster that is entirely in memory. SSDs can greatly mitigate the time required to page data in and out of memory. + +# Provide Columns Names in Lexicographic Order for Best Results + +Although Druid supports schemaless ingestion of dimensions, because of https://github.com/metamx/druid/issues/658, you may sometimes get bigger segments than necessary. To ensure segments are as compact as possible, providing dimension names in lexicographic order is recommended. This may require some ETL processing on your data however. + diff --git a/docs/content/Examples.md b/docs/content/Examples.md index b61168e20dc..dfd44ffe927 100644 --- a/docs/content/Examples.md +++ b/docs/content/Examples.md @@ -19,13 +19,13 @@ Clone Druid and build it: git clone https://github.com/metamx/druid.git druid cd druid git fetch --tags -git checkout druid-0.6.158 +git checkout druid-0.6.159 ./build.sh ``` ### Downloading the DSK (Druid Standalone Kit) -[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.158-bin.tar.gz) a stand-alone tarball and run it: +[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.159-bin.tar.gz) a stand-alone tarball and run it: ``` bash tar -xzf druid-services-0.X.X-bin.tar.gz diff --git a/docs/content/Kafka-Eight.md b/docs/content/Kafka-Eight.md index b7ae4e5c4ab..5819a931b2a 100644 --- a/docs/content/Kafka-Eight.md +++ b/docs/content/Kafka-Eight.md @@ -8,9 +8,9 @@ The previous examples are for Kafka 7. To support Kafka 8, a couple changes need - Update realtime node's configs for Kafka 8 extensions - e.g. - - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-seven:0.6.158",...]` + - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-seven:0.6.159",...]` - becomes - - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-eight:0.6.158",...]` + - `druid.extensions.coordinates=[...,"io.druid.extensions:druid-kafka-eight:0.6.159",...]` - Update realtime task config for changed keys - `firehose.type`, `plumber.rejectionPolicyFactory`, and all of `firehose.consumerProps` changes. diff --git a/docs/content/Production-Cluster-Configuration.md b/docs/content/Production-Cluster-Configuration.md index 70b4cb4036d..6fc319712de 100644 --- a/docs/content/Production-Cluster-Configuration.md +++ b/docs/content/Production-Cluster-Configuration.md @@ -57,7 +57,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/overlord -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.159"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -139,7 +139,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/middlemanager -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.158","io.druid.extensions:druid-kafka-seven:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.159","io.druid.extensions:druid-kafka-seven:0.6.159"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -286,7 +286,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/historical -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.159"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Realtime-Config.md b/docs/content/Realtime-Config.md index c61a22b736d..12ab55ff8e3 100644 --- a/docs/content/Realtime-Config.md +++ b/docs/content/Realtime-Config.md @@ -27,7 +27,7 @@ druid.host=localhost druid.service=realtime druid.port=8083 -druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.159"] druid.zk.service.host=localhost @@ -76,7 +76,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/realtime -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.158","io.druid.extensions:druid-kafka-seven:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.159","io.druid.extensions:druid-kafka-seven:0.6.159"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Router.md b/docs/content/Router.md index 2a077e944fb..73849164216 100644 --- a/docs/content/Router.md +++ b/docs/content/Router.md @@ -51,7 +51,7 @@ druid.service=druid/prod/router druid.extensions.remoteRepositories=[] druid.extensions.localRepository=lib -druid.extensions.coordinates=["io.druid.extensions:druid-histogram:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-histogram:0.6.159"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Simple-Cluster-Configuration.md b/docs/content/Simple-Cluster-Configuration.md index 98871836233..8bfade9ab9a 100644 --- a/docs/content/Simple-Cluster-Configuration.md +++ b/docs/content/Simple-Cluster-Configuration.md @@ -28,7 +28,7 @@ Configuration: -Ddruid.zk.service.host=localhost --Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.158"] +-Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.159"] -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid -Ddruid.db.connector.user=druid diff --git a/docs/content/Thanks.md b/docs/content/Thanks.md deleted file mode 100644 index 97ec7e0904a..00000000000 --- a/docs/content/Thanks.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -layout: doc_page ---- - -YourKit supports the Druid open source projects with its -full-featured Java Profiler. -YourKit, LLC is the creator of innovative and intelligent tools for profiling -Java and .NET applications. Take a look at YourKit's software products: -YourKit Java -Profiler and -YourKit .NET -Profiler. diff --git a/docs/content/Tutorial:-A-First-Look-at-Druid.md b/docs/content/Tutorial:-A-First-Look-at-Druid.md index 7d07d713fb0..4a74b4f5b20 100644 --- a/docs/content/Tutorial:-A-First-Look-at-Druid.md +++ b/docs/content/Tutorial:-A-First-Look-at-Druid.md @@ -49,7 +49,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu ### Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.158-bin.tar.gz). Download this file to a directory of your choosing. +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.159-bin.tar.gz). Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -60,7 +60,7 @@ tar -zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.158 +cd druid-services-0.6.159 ``` You should see a bunch of files: diff --git a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md index e4242e32cbc..7ef8648cc0f 100644 --- a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md +++ b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md @@ -91,7 +91,7 @@ druid.service=overlord druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.159"] druid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid druid.db.connector.user=druid diff --git a/docs/content/Tutorial:-The-Druid-Cluster.md b/docs/content/Tutorial:-The-Druid-Cluster.md index 51e585e40ba..3797746ca20 100644 --- a/docs/content/Tutorial:-The-Druid-Cluster.md +++ b/docs/content/Tutorial:-The-Druid-Cluster.md @@ -13,7 +13,7 @@ In this tutorial, we will set up other types of Druid nodes and external depende If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first. -You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.158-bin.tar.gz) +You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.159-bin.tar.gz) and untar the contents within by issuing: @@ -149,7 +149,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.159"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b @@ -240,7 +240,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.158","io.druid.extensions:druid-kafka-seven:0.6.158"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.159","io.druid.extensions:druid-kafka-seven:0.6.159"] # Change this config to db to hand off to the rest of the Druid cluster druid.publish.type=noop diff --git a/docs/content/Tutorial:-Webstream.md b/docs/content/Tutorial:-Webstream.md index f7893348ef6..47ebec7e049 100644 --- a/docs/content/Tutorial:-Webstream.md +++ b/docs/content/Tutorial:-Webstream.md @@ -37,7 +37,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu h3. Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.158-bin.tar.gz) +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.159-bin.tar.gz) Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -48,7 +48,7 @@ tar zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.158 +cd druid-services-0.6.159 ``` You should see a bunch of files: diff --git a/docs/content/Twitter-Tutorial.md b/docs/content/Twitter-Tutorial.md index dd69a952ae9..95993732449 100644 --- a/docs/content/Twitter-Tutorial.md +++ b/docs/content/Twitter-Tutorial.md @@ -9,7 +9,7 @@ There are two ways to setup Druid: download a tarball, or build it from source. # Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.158-bin.tar.gz). +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.159-bin.tar.gz). Download this bad boy to a directory of your choosing. You can extract the awesomeness within by issuing: diff --git a/docs/content/toc.textile b/docs/content/toc.textile index cc3c86eac99..21f867bca36 100644 --- a/docs/content/toc.textile +++ b/docs/content/toc.textile @@ -19,6 +19,7 @@ h2. Booting a Druid Cluster * "Production Cluster Configuration":Production-Cluster-Configuration.html * "Production Hadoop Configuration":Hadoop-Configuration.html * "Rolling Cluster Updates":Rolling-Updates.html +* "Best Practices":Best-Practices.html h2. Configuration * "Common Configuration":Configuration.html @@ -93,4 +94,4 @@ h2. Development * "Libraries":./Libraries.html h2. Misc -* "Thanks":./Thanks.html +* "Thanks":/thanks.html diff --git a/examples/pom.xml b/examples/pom.xml index e8e39269e2e..fb2597326e7 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index a1fbad02890..6cb8f84483e 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/histogram/pom.xml b/histogram/pom.xml index 2d13ca77b1b..a26ce5f5601 100644 --- a/histogram/pom.xml +++ b/histogram/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java index 73f0a562c06..1c7b23b3a3f 100644 --- a/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java +++ b/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java @@ -30,12 +30,9 @@ import io.druid.data.input.Row; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.aggregation.MaxAggregatorFactory; -import io.druid.query.aggregation.MinAggregatorFactory; import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; -import io.druid.query.dimension.LegacyDimensionSpec; import io.druid.query.groupby.GroupByQuery; import io.druid.query.groupby.GroupByQueryConfig; import io.druid.query.groupby.GroupByQueryEngine; @@ -45,7 +42,6 @@ import io.druid.query.groupby.GroupByQueryRunnerTestHelper; import io.druid.query.groupby.orderby.DefaultLimitSpec; import io.druid.query.groupby.orderby.OrderByColumnSpec; import io.druid.segment.TestHelper; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -158,34 +154,33 @@ public class ApproximateHistogramGroupByQueryTest GroupByQuery query = new GroupByQuery.Builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setGranularity(QueryRunnerTestHelper.allGran) - .setDimensions(Arrays.asList(new LegacyDimensionSpec(QueryRunnerTestHelper.marketDimension))) + .setDimensions( + Arrays.asList( + new DefaultDimensionSpec( + QueryRunnerTestHelper.marketDimension, + "marKetAlias" + ) + ) + ) .setInterval(QueryRunnerTestHelper.fullOnInterval) .setLimitSpec( new DefaultLimitSpec( Lists.newArrayList( new OrderByColumnSpec( - QueryRunnerTestHelper.marketDimension, - OrderByColumnSpec.Direction.ASCENDING + "marKetAlias", + OrderByColumnSpec.Direction.DESCENDING ) ), 1 ) ) .setAggregatorSpecs( Lists.newArrayList( - Iterables.concat( - QueryRunnerTestHelper.commonAggregators, - Lists.newArrayList( - new MaxAggregatorFactory("maxIndex", "index"), - new MinAggregatorFactory("minIndex", "index"), - aggFactory - ) - ) + QueryRunnerTestHelper.rowsCount, + aggFactory ) ) .setPostAggregatorSpecs( - Arrays.asList( - QueryRunnerTestHelper.addRowsIndexConstant, - QueryRunnerTestHelper.dependentPostAgg, + Arrays.asList( new QuantilePostAggregator("quantile", "apphisto", 0.5f) ) ) @@ -194,31 +189,21 @@ public class ApproximateHistogramGroupByQueryTest List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", - "market", "spot", - "rows", 837L, - "addRowsIndexConstant", 96444.5703125, - "dependentPostAgg", 97282.5703125, - "index", 95606.5703125, - "maxIndex", 277.2735290527344, - "minIndex", 59.02102279663086, - "quantile", 101.78856f, - "uniques", QueryRunnerTestHelper.UNIQUES_9, + "marketalias", "upfront", + "rows", 186L, + "quantile", 880.9881f, "apphisto", new Histogram( new float[]{ - 4.457897186279297f, - 59.02102279663086f, - 113.58415222167969f, - 168.14727783203125f, - 222.7104034423828f, - 277.2735290527344f + 214.97299194335938f, + 545.9906005859375f, + 877.0081787109375f, + 1208.0257568359375f, + 1539.0433349609375f, + 1870.06103515625f }, new double[]{ - 0.0, - 462.4309997558594, - 357.5404968261719, - 15.022850036621094, - 2.0056631565093994 + 0.0, 67.53287506103516, 72.22068786621094, 31.984678268432617, 14.261756896972656 } ) ) diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 7ae737f572f..cfd2e8da529 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index bb18a62d312..5fe0ebae569 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java index f657b6a2435..f78f576de67 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ImmutableZkWorker.java @@ -32,7 +32,7 @@ public class ImmutableZkWorker { private final Worker worker; private final int currCapacityUsed; - private final Set availabilityGroups; + private final ImmutableSet availabilityGroups; public ImmutableZkWorker(Worker worker, int currCapacityUsed, Set availabilityGroups) { diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java index e9bba479c64..e382c6dee3c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java @@ -164,7 +164,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer @Override public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception { - Worker worker; + final Worker worker; switch (event.getType()) { case CHILD_ADDED: worker = jsonMapper.readValue( @@ -198,6 +198,14 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer } ); break; + case CHILD_UPDATED: + worker = jsonMapper.readValue( + event.getData().getData(), + Worker.class + ); + updateWorker(worker); + break; + case CHILD_REMOVED: worker = jsonMapper.readValue( event.getData().getData(), @@ -745,6 +753,24 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer } } + /** + * We allow workers to change their own capacities and versions. They cannot change their own hosts or ips without + * dropping themselves and re-announcing. + */ + private void updateWorker(final Worker worker) + { + final ZkWorker zkWorker = zkWorkers.get(worker.getHost()); + if (zkWorker != null) { + log.info("Worker[%s] updated its announcement from[%s] to[%s].", worker.getHost(), zkWorker.getWorker(), worker); + zkWorker.setWorker(worker); + } else { + log.warn( + "WTF, worker[%s] updated its announcement but we didn't have a ZkWorker for it. Ignoring.", + worker.getHost() + ); + } + } + /** * When a ephemeral worker node disappears from ZK, incomplete running tasks will be retried by * the logic in the status listener. We still have to make sure there are no tasks assigned diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java index abc4da0ad57..54b09da2b29 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ZkWorker.java @@ -22,11 +22,11 @@ package io.druid.indexing.overlord; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import io.druid.indexing.common.task.Task; import io.druid.indexing.worker.TaskAnnouncement; import io.druid.indexing.worker.Worker; import org.apache.curator.framework.recipes.cache.ChildData; @@ -46,15 +46,15 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ZkWorker implements Closeable { - private final Worker worker; private final PathChildrenCache statusCache; private final Function cacheConverter; + private AtomicReference worker; private AtomicReference lastCompletedTaskTime = new AtomicReference(new DateTime()); public ZkWorker(Worker worker, PathChildrenCache statusCache, final ObjectMapper jsonMapper) { - this.worker = worker; + this.worker = new AtomicReference<>(worker); this.statusCache = statusCache; this.cacheConverter = new Function() { @@ -84,7 +84,7 @@ public class ZkWorker implements Closeable @JsonProperty("worker") public Worker getWorker() { - return worker; + return worker.get(); } @JsonProperty("runningTasks") @@ -137,30 +137,28 @@ public class ZkWorker implements Closeable return getRunningTasks().containsKey(taskId); } - public boolean isAtCapacity() - { - return getCurrCapacityUsed() >= worker.getCapacity(); - } - public boolean isValidVersion(String minVersion) { - return worker.getVersion().compareTo(minVersion) >= 0; + return worker.get().getVersion().compareTo(minVersion) >= 0; } - public boolean canRunTask(Task task) + public void setWorker(Worker newWorker) { - return (worker.getCapacity() - getCurrCapacityUsed() >= task.getTaskResource().getRequiredCapacity() - && !getAvailabilityGroups().contains(task.getTaskResource().getAvailabilityGroup())); + final Worker oldWorker = worker.get(); + Preconditions.checkArgument(newWorker.getHost().equals(oldWorker.getHost()), "Cannot change Worker host"); + Preconditions.checkArgument(newWorker.getIp().equals(oldWorker.getIp()), "Cannot change Worker ip"); + + worker.set(newWorker); } public void setLastCompletedTaskTime(DateTime completedTaskTime) { - lastCompletedTaskTime.getAndSet(completedTaskTime); + lastCompletedTaskTime.set(completedTaskTime); } public ImmutableZkWorker toImmutable() { - return new ImmutableZkWorker(worker, getCurrCapacityUsed(), getAvailabilityGroups()); + return new ImmutableZkWorker(worker.get(), getCurrCapacityUsed(), getAvailabilityGroups()); } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index 11c7c85c639..b5f86ea7494 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -361,6 +361,29 @@ public class RemoteTaskRunnerTest Assert.assertEquals(TaskStatus.Status.FAILED, status.getStatusCode()); } + @Test + public void testWorkerDisabled() throws Exception + { + doSetup(); + final ListenableFuture result = remoteTaskRunner.run(task); + + Assert.assertTrue(taskAnnounced(task.getId())); + mockWorkerRunningTask(task); + Assert.assertTrue(workerRunningTask(task.getId())); + + // Disable while task running + disableWorker(); + + // Continue test + mockWorkerCompleteSuccessfulTask(task); + Assert.assertTrue(workerCompletedTask(result)); + Assert.assertEquals(task.getId(), result.get().getId()); + Assert.assertEquals(TaskStatus.Status.SUCCESS, result.get().getStatusCode()); + + // Confirm RTR thinks the worker is disabled. + Assert.assertEquals("", Iterables.getOnlyElement(remoteTaskRunner.getWorkers()).getWorker().getVersion()); + } + private void doSetup() throws Exception { makeWorker(); @@ -405,6 +428,14 @@ public class RemoteTaskRunnerTest ); } + private void disableWorker() throws Exception + { + cf.setData().forPath( + announcementsPath, + jsonMapper.writeValueAsBytes(new Worker(worker.getHost(), worker.getIp(), worker.getCapacity(), "")) + ); + } + private boolean taskAnnounced(final String taskId) { return pathExists(joiner.join(tasksPath, taskId)); diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index d6fa98d745a..be87d5b4d79 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 66f80d94371..a26e940b725 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/pom.xml b/pom.xml index 382fa139925..0c6a4b34aa8 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ io.druid druid pom - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT druid druid @@ -89,7 +89,7 @@ com.metamx bytebuffer-collections - 0.0.2 + 0.0.4 com.metamx @@ -194,7 +194,7 @@ it.uniroma3.mat extendedset - 1.3.4 + 1.3.7 com.google.guava diff --git a/processing/pom.xml b/processing/pom.xml index adfec8434f8..867b720ca57 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index b1db05a2284..071a91b3c3e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -63,12 +63,12 @@ public class CardinalityAggregatorFactory implements AggregatorFactory public CardinalityAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldNames") final List fieldNames, - @JsonProperty("byRow") final Boolean byRow + @JsonProperty("byRow") final boolean byRow ) { this.name = name; this.fieldNames = fieldNames; - this.byRow = byRow == null ? false : byRow; + this.byRow = byRow; } @Override @@ -203,6 +203,12 @@ public class CardinalityAggregatorFactory implements AggregatorFactory return fieldNames; } + @JsonProperty + public boolean isByRow() + { + return byRow; + } + @Override public byte[] getCacheKey() { diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index bf6fc138ad4..ad96280a925 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -173,7 +173,7 @@ public class DefaultLimitSpec implements LimitSpec public String apply(Row input) { // Multi-value dimensions have all been flattened at this point; - final List dimList = input.getDimension(dimension); + final List dimList = input.getDimension(dimension.toLowerCase()); return dimList.isEmpty() ? null : dimList.get(0); } } diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 2027e1bc096..bb0556f4757 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -82,7 +82,7 @@ public class QueryRunnerTestHelper ); public static final QueryGranularity dayGran = QueryGranularity.DAY; public static final QueryGranularity allGran = QueryGranularity.ALL; - public static final String marketDimension = "market"; + public static final String marketDimension = "marKet"; public static final String qualityDimension = "quality"; public static final String placementDimension = "placement"; public static final String placementishDimension = "placementish"; diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 4d27df07303..9d8ac7c721f 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -19,16 +19,20 @@ package io.druid.query.aggregation.cardinality; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.jackson.DefaultObjectMapper; import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; -import junit.framework.Assert; +import org.junit.Assert; import org.junit.Test; import javax.annotation.Nullable; @@ -378,4 +382,15 @@ public class CardinalityAggregatorTest 0.05 ); } + + @Test + public void testSerde() throws Exception + { + CardinalityAggregatorFactory factory = new CardinalityAggregatorFactory("billy", ImmutableList.of("b", "a", "c"), true); + ObjectMapper objectMapper = new DefaultObjectMapper(); + Assert.assertEquals( + factory, + objectMapper.readValue(objectMapper.writeValueAsString(factory), AggregatorFactory.class) + ); + } } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 5be3210e5ac..772d3b0d18d 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -733,6 +733,48 @@ public class GroupByQueryRunnerTest ); } + @Test + public void testGroupByWithMixedCasingOrdering() + { + GroupByQuery query = new GroupByQuery.Builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setGranularity(QueryRunnerTestHelper.allGran) + .setDimensions( + Arrays.asList( + new DefaultDimensionSpec( + QueryRunnerTestHelper.marketDimension, + "MarketAlias" + ) + ) + ) + .setInterval(QueryRunnerTestHelper.fullOnInterval) + .setLimitSpec( + new DefaultLimitSpec( + Lists.newArrayList( + new OrderByColumnSpec( + "marketALIAS", + OrderByColumnSpec.Direction.DESCENDING + ) + ), 3 + ) + ) + .setAggregatorSpecs( + Lists.newArrayList( + QueryRunnerTestHelper.rowsCount + ) + ) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "marketalias", "upfront", "rows", 186L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "marketalias", "total_market", "rows", 186L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "marketalias", "spot", "rows", 837L) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "order-limit"); + } + @Test public void testHavingSpec() { diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index c7ffa9270ae..745f10e88ac 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 9fcaf6349dd..e4914713493 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index fe3b96333e2..740e06d56ad 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index f9c32a441a2..9968d0b6146 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.159-SNAPSHOT + 0.6.160-SNAPSHOT