From 3b6c647b1b86f43625f55a4b9b50a9f9d86fcba7 Mon Sep 17 00:00:00 2001 From: Tugdual Saunier Date: Wed, 28 May 2014 15:54:53 +0100 Subject: [PATCH 01/39] Fixed maven build when druid is integrated as a submodule --- services/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/services/pom.xml b/services/pom.xml index 80c4b4add81..16b02c6bb8c 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -101,10 +101,6 @@ distro-assembly - package - - assembly - src/assembly/assembly.xml From bf09f979bd5f4e567d5650d879a7d3addc956e10 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 5 Jun 2014 20:35:57 +0530 Subject: [PATCH 02/39] fix segment metadataquery --- .../src/main/java/io/druid/query/Druids.java | 110 ++++++++++++++++++ .../SegmentMetadataQueryRunnerFactory.java | 10 +- .../metadata/SegmentMetadataQueryTest.java | 59 ++++++++++ 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index 87676414ab0..084652b322f 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -29,6 +29,8 @@ import io.druid.query.filter.NoopDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; import io.druid.query.filter.SelectorDimFilter; +import io.druid.query.metadata.metadata.ColumnIncluderator; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; import io.druid.query.search.SearchResultValue; import io.druid.query.search.search.InsensitiveContainsSearchQuerySpec; import io.druid.query.search.search.SearchQuery; @@ -823,4 +825,112 @@ public class Druids { return new ResultBuilder(); } + + /** + * A Builder for SegmentMetadataQuery. + *

+ * Required: dataSource(), intervals() must be called before build() + *

+ * Usage example: + * 


+   *   SegmentMetadataQuery query = new SegmentMetadataQueryBuilder()
+   *                                  .dataSource("Example")
+   *                                  .interval("2010/2013")
+   *                                  .build();
+   *
+ * + * @see io.druid.query.metadata.metadata.SegmentMetadataQuery + */ + public static class SegmentMetadataQueryBuilder + { + private DataSource dataSource; + private QuerySegmentSpec querySegmentSpec; + private ColumnIncluderator toInclude; + private Boolean merge; + private Map context; + + public SegmentMetadataQueryBuilder() + { + dataSource = null; + querySegmentSpec = null; + toInclude = null; + merge = null; + context = null; + } + + public SegmentMetadataQuery build() + { + return new SegmentMetadataQuery( + dataSource, + querySegmentSpec, + toInclude, + merge, + context + ); + } + + public SegmentMetadataQueryBuilder copy(SegmentMetadataQueryBuilder builder) + { + return new SegmentMetadataQueryBuilder() + .dataSource(builder.dataSource) + .intervals(builder.querySegmentSpec) + .toInclude(toInclude) + .merge(merge) + .context(builder.context); + } + + public SegmentMetadataQueryBuilder dataSource(String ds) + { + dataSource = new TableDataSource(ds); + return this; + } + + public SegmentMetadataQueryBuilder dataSource(DataSource ds) + { + dataSource = ds; + return this; + } + + public SegmentMetadataQueryBuilder intervals(QuerySegmentSpec q) + { + querySegmentSpec = q; + return this; + } + + public SegmentMetadataQueryBuilder intervals(String s) + { + querySegmentSpec = new LegacySegmentSpec(s); + return this; + } + + public SegmentMetadataQueryBuilder intervals(List l) + { + querySegmentSpec = new LegacySegmentSpec(l); + return this; + } + + public SegmentMetadataQueryBuilder toInclude(ColumnIncluderator toInclude) + { + this.toInclude = toInclude; + return this; + } + + + public SegmentMetadataQueryBuilder merge(boolean merge) + { + this.merge = merge; + return this; + } + + public SegmentMetadataQueryBuilder context(Map c) + { + context = c; + return this; + } + } + + public static SegmentMetadataQueryBuilder newSegmentMetadataQueryBuilder() + { + return new SegmentMetadataQueryBuilder(); + } } diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index c5b64c2d46a..9fa393c5c2f 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -25,6 +25,7 @@ import com.google.common.collect.Maps; import com.metamx.common.guava.ExecutorExecutingSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import io.druid.query.AbstractPrioritizedCallable; import io.druid.query.ConcatQueryRunner; import io.druid.query.Query; import io.druid.query.QueryRunner; @@ -116,17 +117,14 @@ public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory run(final Query query) { - + final int priority = query.getContextPriority(0); Future> future = queryExecutor.submit( - new Callable>() + new AbstractPrioritizedCallable>(priority) { @Override public Sequence call() throws Exception { - return new ExecutorExecutingSequence( - input.run(query), - queryExecutor - ); + return input.run(query); } } ); diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index 6e6dd9ce275..5d596292348 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -21,17 +21,76 @@ package io.druid.query.metadata; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequences; import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.metadata.metadata.ColumnAnalysis; +import io.druid.query.metadata.metadata.ListColumnIncluderator; +import io.druid.query.metadata.metadata.SegmentAnalysis; import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.TestIndex; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; +import java.util.Arrays; + public class SegmentMetadataQueryTest { + @SuppressWarnings("unchecked") + private final QueryRunner runner = makeQueryRunner( + new SegmentMetadataQueryRunnerFactory() + ); private ObjectMapper mapper = new DefaultObjectMapper(); + @SuppressWarnings("unchecked") + public static QueryRunner makeQueryRunner( + QueryRunnerFactory factory + ) + { + return QueryRunnerTestHelper.makeQueryRunner( + factory, + new QueryableIndexSegment(QueryRunnerTestHelper.segmentId, TestIndex.getMMappedTestIndex()) + ); + } + + @Test + @SuppressWarnings("unchecked") + public void testSegmentMetadataQuery() + { + SegmentMetadataQuery query = Druids.newSegmentMetadataQueryBuilder() + .dataSource("testing") + .intervals("2013/2014") + .toInclude(new ListColumnIncluderator(Arrays.asList("placement"))) + .merge(true) + .build(); + + Iterable results = Sequences.toList( + runner.run(query), + Lists.newArrayList() + ); + SegmentAnalysis val = results.iterator().next(); + Assert.assertEquals("testSegment", val.getId()); + Assert.assertEquals(69843, val.getSize()); + Assert.assertEquals( + Arrays.asList(new Interval("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), + val.getIntervals() + ); + Assert.assertEquals(1, val.getColumns().size()); + final ColumnAnalysis columnAnalysis = val.getColumns().get("placement"); + Assert.assertEquals("STRING", columnAnalysis.getType()); + Assert.assertEquals(10881, columnAnalysis.getSize()); + Assert.assertEquals(new Integer(1), columnAnalysis.getCardinality()); + Assert.assertNull(columnAnalysis.getErrorMessage()); + + } + @Test public void testSerde() throws Exception { From c8689507c8bae7fd3c03a326f0836554da37adef Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 5 Jun 2014 10:01:25 -0700 Subject: [PATCH 03/39] prepare for next release --- docs/content/Examples.md | 4 ++-- docs/content/Production-Cluster-Configuration.md | 6 +++--- docs/content/Realtime-Config.md | 4 ++-- docs/content/Simple-Cluster-Configuration.md | 2 +- docs/content/Tutorial:-A-First-Look-at-Druid.md | 4 ++-- docs/content/Tutorial:-Loading-Your-Data-Part-1.md | 2 +- docs/content/Tutorial:-The-Druid-Cluster.md | 6 +++--- docs/content/Tutorial:-Webstream.md | 4 ++-- docs/content/Twitter-Tutorial.md | 2 +- examples/config/historical/runtime.properties | 2 +- examples/config/overlord/runtime.properties | 2 +- examples/config/realtime/runtime.properties | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/content/Examples.md b/docs/content/Examples.md index 268f655bc05..58fb1f45e01 100644 --- a/docs/content/Examples.md +++ b/docs/content/Examples.md @@ -19,13 +19,13 @@ Clone Druid and build it: git clone https://github.com/metamx/druid.git druid cd druid git fetch --tags -git checkout druid-0.6.119 +git checkout druid-0.6.121 ./build.sh ``` ### Downloading the DSK (Druid Standalone Kit) -[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.119-bin.tar.gz) a stand-alone tarball and run it: +[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz) a stand-alone tarball and run it: ``` bash tar -xzf druid-services-0.X.X-bin.tar.gz diff --git a/docs/content/Production-Cluster-Configuration.md b/docs/content/Production-Cluster-Configuration.md index 413cfd597c4..76c481ec7bc 100644 --- a/docs/content/Production-Cluster-Configuration.md +++ b/docs/content/Production-Cluster-Configuration.md @@ -55,7 +55,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/overlord -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -137,7 +137,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/middlemanager -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119","io.druid.extensions:druid-kafka-seven:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121","io.druid.extensions:druid-kafka-seven:0.6.121"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod @@ -285,7 +285,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/historical -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Realtime-Config.md b/docs/content/Realtime-Config.md index 62dfcf4a21f..a44c94fbdfa 100644 --- a/docs/content/Realtime-Config.md +++ b/docs/content/Realtime-Config.md @@ -27,7 +27,7 @@ druid.host=localhost druid.service=realtime druid.port=8083 -druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.121"] druid.zk.service.host=localhost @@ -76,7 +76,7 @@ druid.host=#{IP_ADDR}:8080 druid.port=8080 druid.service=druid/prod/realtime -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119","io.druid.extensions:druid-kafka-seven:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121","io.druid.extensions:druid-kafka-seven:0.6.121"] druid.zk.service.host=#{ZK_IPs} druid.zk.paths.base=/druid/prod diff --git a/docs/content/Simple-Cluster-Configuration.md b/docs/content/Simple-Cluster-Configuration.md index 3ea9f9b20a7..51bc7778fd9 100644 --- a/docs/content/Simple-Cluster-Configuration.md +++ b/docs/content/Simple-Cluster-Configuration.md @@ -28,7 +28,7 @@ Configuration: -Ddruid.zk.service.host=localhost --Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.119"] +-Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.121"] -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid -Ddruid.db.connector.user=druid diff --git a/docs/content/Tutorial:-A-First-Look-at-Druid.md b/docs/content/Tutorial:-A-First-Look-at-Druid.md index a27d7277e2d..0afd5a36416 100644 --- a/docs/content/Tutorial:-A-First-Look-at-Druid.md +++ b/docs/content/Tutorial:-A-First-Look-at-Druid.md @@ -49,7 +49,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu ### Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.119-bin.tar.gz). Download this file to a directory of your choosing. +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz). Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -60,7 +60,7 @@ tar -zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.119 +cd druid-services-0.6.121 ``` You should see a bunch of files: diff --git a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md index 0427550eeed..eb1d71c35c3 100644 --- a/docs/content/Tutorial:-Loading-Your-Data-Part-1.md +++ b/docs/content/Tutorial:-Loading-Your-Data-Part-1.md @@ -96,7 +96,7 @@ The configurations for the overlord node are as follows: -Ddruid.zk.service.host=localhost --Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.119"] +-Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.121"] -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid -Ddruid.db.connector.user=druid diff --git a/docs/content/Tutorial:-The-Druid-Cluster.md b/docs/content/Tutorial:-The-Druid-Cluster.md index d63f7d70e6f..259e4d3eb07 100644 --- a/docs/content/Tutorial:-The-Druid-Cluster.md +++ b/docs/content/Tutorial:-The-Druid-Cluster.md @@ -13,7 +13,7 @@ In this tutorial, we will set up other types of Druid nodes and external depende If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first. -You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.119-bin.tar.gz) +You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz) and untar the contents within by issuing: @@ -149,7 +149,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b @@ -240,7 +240,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.119","io.druid.extensions:druid-kafka-seven:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.121","io.druid.extensions:druid-kafka-seven:0.6.121"] # Change this config to db to hand off to the rest of the Druid cluster druid.publish.type=noop diff --git a/docs/content/Tutorial:-Webstream.md b/docs/content/Tutorial:-Webstream.md index abd0edda56f..3d8d70e8fcd 100644 --- a/docs/content/Tutorial:-Webstream.md +++ b/docs/content/Tutorial:-Webstream.md @@ -37,7 +37,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu h3. Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.119-bin.tar.gz) +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz) Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -48,7 +48,7 @@ tar zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.119 +cd druid-services-0.6.121 ``` You should see a bunch of files: diff --git a/docs/content/Twitter-Tutorial.md b/docs/content/Twitter-Tutorial.md index 857c32f6256..a76ed397e65 100644 --- a/docs/content/Twitter-Tutorial.md +++ b/docs/content/Twitter-Tutorial.md @@ -9,7 +9,7 @@ There are two ways to setup Druid: download a tarball, or build it from source. # Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.119-bin.tar.gz). +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz). Download this bad boy to a directory of your choosing. You can extract the awesomeness within by issuing: diff --git a/examples/config/historical/runtime.properties b/examples/config/historical/runtime.properties index f86944fd020..fd0cbfc520a 100644 --- a/examples/config/historical/runtime.properties +++ b/examples/config/historical/runtime.properties @@ -4,7 +4,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.121"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b diff --git a/examples/config/overlord/runtime.properties b/examples/config/overlord/runtime.properties index 7a8582ed4d3..33cf72d3cf4 100644 --- a/examples/config/overlord/runtime.properties +++ b/examples/config/overlord/runtime.properties @@ -9,7 +9,7 @@ -Ddruid.zk.service.host=localhost --Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.119"] +-Ddruid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.121"] -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid -Ddruid.db.connector.user=druid diff --git a/examples/config/realtime/runtime.properties b/examples/config/realtime/runtime.properties index 43e96fbe086..07c22756b6b 100644 --- a/examples/config/realtime/runtime.properties +++ b/examples/config/realtime/runtime.properties @@ -4,7 +4,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.119","io.druid.extensions:druid-kafka-seven:0.6.119","io.druid.extensions:druid-rabbitmq:0.6.119"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.121","io.druid.extensions:druid-kafka-seven:0.6.121","io.druid.extensions:druid-rabbitmq:0.6.121"] # Change this config to db to hand off to the rest of the Druid cluster druid.publish.type=noop From 0265ae9ac0890aa38ce7e5b687a7842f876c7f9e Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 5 Jun 2014 10:03:26 -0700 Subject: [PATCH 04/39] [maven-release-plugin] prepare release druid-0.6.121 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 754e227b4c1..8fecb85bafb 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/common/pom.xml b/common/pom.xml index e76f3945f2e..d60196183dd 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/examples/pom.xml b/examples/pom.xml index 7a240100744..8a1b0d999e4 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 9de4f7a5b97..d7f0b803b58 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 7b04b613762..b08f88069e8 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 5b83d487159..97f3407c778 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 9ed51e530bd..2a650af810e 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index dff8aada79d..63a38cb7275 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/pom.xml b/pom.xml index 12bfd8df636..c3a1ac5f3aa 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.121-SNAPSHOT + 0.6.121 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.107-SNAPSHOT + druid-0.6.121 diff --git a/processing/pom.xml b/processing/pom.xml index e714babd8cf..9accd2403aa 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 8345cfc1007..5982e0817e9 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 967c9f70422..33b3b118052 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/server/pom.xml b/server/pom.xml index e097487ea02..3b72c33b6b4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 diff --git a/services/pom.xml b/services/pom.xml index 3787e7a6b96..a1dd3019c39 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.121-SNAPSHOT + 0.6.121 From 46ef5c4560fb576d3e750ab35b6ca1cb05626a01 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 5 Jun 2014 10:03:31 -0700 Subject: [PATCH 05/39] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 8fecb85bafb..86ecc759553 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index d60196183dd..4e828108927 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 8a1b0d999e4..5b9cbc47847 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index d7f0b803b58..46b6a44ce6d 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index b08f88069e8..6decbae2e15 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 97f3407c778..9db9badd919 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 2a650af810e..5840e57e491 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 63a38cb7275..25bfd488014 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/pom.xml b/pom.xml index c3a1ac5f3aa..2116d7f617f 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.121 + 0.6.122-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.121 + druid-0.6.107-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index 9accd2403aa..f23eb94be46 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 5982e0817e9..19a463d66e9 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 33b3b118052..1f6ba89a2cb 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 3b72c33b6b4..83635cdc29f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index a1dd3019c39..75b1fd47474 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.121 + 0.6.122-SNAPSHOT From 03e1fb6f3f3396ec0dc968d6896e6398a82edf9a Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 5 Jun 2014 23:14:51 +0530 Subject: [PATCH 06/39] reduce memory usage for end points do not copy entire segments when not needed, reduces memory pressure on the coordinator when no. of segments is large enough. --- .../server/coordinator/DruidCoordinator.java | 19 ++++++++++++++++++- .../DruidCoordinatorSegmentInfoLoader.java | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index f57bd87ebd5..97c1a3eab61 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -422,7 +422,7 @@ public class DruidCoordinator } } - public Set getAvailableDataSegments() + public Set getOrderedAvailableDataSegments() { Set availableSegments = Sets.newTreeSet(Comparators.inverse(DataSegment.bucketMonthComparator())); @@ -452,6 +452,23 @@ public class DruidCoordinator return availableSegments; } + public Iterable getAvailableDataSegments() + { + return Iterables.concat( + Iterables.transform( + databaseSegmentManager.getInventory(), + new Function>() + { + @Override + public Iterable apply(DruidDataSource input) + { + return input.getSegments(); + } + } + ) + ); + } + @LifecycleStart public void start() { diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentInfoLoader.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentInfoLoader.java index 8980bfded90..ea5485cb2b6 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentInfoLoader.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentInfoLoader.java @@ -41,7 +41,7 @@ public class DruidCoordinatorSegmentInfoLoader implements DruidCoordinatorHelper public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) { // Display info about all available segments - final Set availableSegments = coordinator.getAvailableDataSegments(); + final Set availableSegments = coordinator.getOrderedAvailableDataSegments(); if (log.isDebugEnabled()) { log.debug("Available DataSegments"); for (DataSegment dataSegment : availableSegments) { From ab313005132783be84fe65942ea2ce24652a6a29 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 5 Jun 2014 23:16:38 +0530 Subject: [PATCH 07/39] remove duplicate code --- .../druid/server/coordinator/DruidCoordinator.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 97c1a3eab61..7cbd3bbcafa 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -426,19 +426,7 @@ public class DruidCoordinator { Set availableSegments = Sets.newTreeSet(Comparators.inverse(DataSegment.bucketMonthComparator())); - Iterable dataSegments = Iterables.concat( - Iterables.transform( - databaseSegmentManager.getInventory(), - new Function>() - { - @Override - public Iterable apply(DruidDataSource input) - { - return input.getSegments(); - } - } - ) - ); + Iterable dataSegments = getAvailableDataSegments(); for (DataSegment dataSegment : dataSegments) { if (dataSegment.getSize() < 0) { From 5ef660f6d1cbf8c51aa4daf49ad68ccae96fc912 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Fri, 6 Jun 2014 00:23:34 +0530 Subject: [PATCH 08/39] improve mem usage for getLoadStatus --- .../server/coordinator/DruidCoordinator.java | 53 +++++-------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 7cbd3bbcafa..0c17aa54fa1 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -77,6 +77,7 @@ import org.joda.time.Duration; import java.io.IOException; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -91,11 +92,8 @@ import java.util.concurrent.atomic.AtomicReference; public class DruidCoordinator { public static final String COORDINATOR_OWNER_NODE = "_COORDINATOR"; - private static final EmittingLogger log = new EmittingLogger(DruidCoordinator.class); - private final Object lock = new Object(); - private final DruidCoordinatorConfig config; private final ZkPathsConfig zkPaths; private final JacksonConfigManager configManager; @@ -111,7 +109,6 @@ public class DruidCoordinator private final AtomicReference leaderLatch; private final ServiceAnnouncer serviceAnnouncer; private final DruidNode self; - private volatile boolean started = false; private volatile int leaderCounter = 0; private volatile boolean leader = false; @@ -234,7 +231,6 @@ public class DruidCoordinator return retVal; } - public CountingMap getSegmentAvailability() { final CountingMap retVal = new CountingMap<>(); @@ -253,43 +249,22 @@ public class DruidCoordinator public Map getLoadStatus() { - // find available segments - Map> availableSegments = Maps.newHashMap(); - for (DataSegment dataSegment : getAvailableDataSegments()) { - Set segments = availableSegments.get(dataSegment.getDataSource()); - if (segments == null) { - segments = Sets.newHashSet(); - availableSegments.put(dataSegment.getDataSource(), segments); - } - segments.add(dataSegment); - } - - // find segments currently loaded - Map> segmentsInCluster = Maps.newHashMap(); - for (DruidServer druidServer : serverInventoryView.getInventory()) { - for (DruidDataSource druidDataSource : druidServer.getDataSources()) { - Set segments = segmentsInCluster.get(druidDataSource.getName()); - if (segments == null) { - segments = Sets.newHashSet(); - segmentsInCluster.put(druidDataSource.getName(), segments); - } - segments.addAll(druidDataSource.getSegments()); - } - } - - // compare available segments with currently loaded Map loadStatus = Maps.newHashMap(); - for (Map.Entry> entry : availableSegments.entrySet()) { - String dataSource = entry.getKey(); - Set segmentsAvailable = entry.getValue(); - Set loadedSegments = segmentsInCluster.get(dataSource); - if (loadedSegments == null) { - loadedSegments = Sets.newHashSet(); + for (DruidDataSource dataSource : databaseSegmentManager.getInventory()) { + final Set segments = Sets.newHashSet(dataSource.getSegments()); + final int availableSegmentSize = segments.size(); + + // remove loaded segments + for (DruidServer druidServer : serverInventoryView.getInventory()) { + final DruidDataSource loadedView = druidServer.getDataSource(dataSource.getName()); + if (loadedView != null) { + segments.removeAll(loadedView.getSegments()); + } } - Set unloadedSegments = Sets.difference(segmentsAvailable, loadedSegments); + final int unloadedSegmentSize = segments.size(); loadStatus.put( - dataSource, - 100 * ((double) (segmentsAvailable.size() - unloadedSegments.size()) / (double) segmentsAvailable.size()) + dataSource.getName(), + 100 * ((double) (availableSegmentSize - unloadedSegmentSize) / (double) availableSegmentSize) ); } From a0d51a8cb234150f72184855147d6b0cadb2f304 Mon Sep 17 00:00:00 2001 From: jisookim0513 Date: Thu, 5 Jun 2014 15:51:23 -0700 Subject: [PATCH 09/39] modify whirr documentation --- docs/content/Booting-a-production-cluster.md | 28 +++++++++++++++---- .../indexing/wikipedia_realtime_task.json | 4 +-- .../src/main/java/io/druid/cli/CliBroker.java | 2 +- .../main/java/io/druid/cli/CliHistorical.java | 2 +- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/content/Booting-a-production-cluster.md b/docs/content/Booting-a-production-cluster.md index cd1205bac23..c97e25cfdee 100644 --- a/docs/content/Booting-a-production-cluster.md +++ b/docs/content/Booting-a-production-cluster.md @@ -12,18 +12,18 @@ You can provision individual servers, loading Druid onto each machine (or buildi [Apache Whirr](http://whirr.apache.org/) is a set of libraries for launching cloud services. For Druid, Whirr serves as an easy way to launch a cluster in Amazon AWS by using simple commands and configuration files (called *recipes*). -**NOTE:** Whirr will install Druid 0.5.x. At this time Whirr can launch Druid 0.5.x only, but in the near future will support Druid 0.6.x. You can download and install your own copy of Druid 0.5.x [here](http://static.druid.io/artifacts/releases/druid-services-0.5.7-bin.tar.gz). +**NOTE:** Whirr will install Druid 0.6.115. Also, it doesn't work with JDK1.7.0_55. JDK1.7.0_45 recommended. -You'll need an AWS account, and an EC2 key pair from that account so that Whirr can connect to the cloud via the EC2 API. If you haven't generated a key pair, see the [AWS documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) or see this [Whirr FAQ](http://whirr.apache.org/faq.html#how-do-i-find-my-cloud-credentials). +You'll need an AWS account, S3 Bucket and an EC2 key pair from that account so that Whirr can connect to the cloud via the EC2 API. If you haven't generated a key pair, see the [AWS documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) or see this [Whirr FAQ](http://whirr.apache.org/faq.html#how-do-i-find-my-cloud-credentials). ### Installing Whirr You must use a version of Whirr that includes and supports a Druid recipe. You can do it so in one of two ways: #### Build the Following Version of Whirr -Clone the code from [https://github.com/rjurney/whirr/tree/trunk](https://github.com/rjurney/whirr/tree/trunk) and build Whirr: +Clone the code from [https://github.com/druid-io/whirr](https://github.com/druid-io/whirr) and build Whirr: - git clone git@github.com:rjurney/whirr.git + git clone git@github.com:druid-io/whirr.git cd whirr git checkout trunk mvn clean install -Dmaven.test.failure.ignore=true @@ -39,10 +39,12 @@ Then run `mvn install` from the root directory. The Whirr recipe for Druid is the configuration file `$WHIRR_HOME/recipies/druid.properties`. You can edit this file to suit your needs -- it is annotated and self-explanatory. Here are some hints about that file: * Set `whirr.location-id` to a specific AWS region (e.g., us-east-1) if desired, else one will be chosen for you. -* You can choose the hardware used with `whirr.hardware-id` to a specific instance type (e.g., m1.large). If you don't choose an image via `whirr.image-id` (image must be compatible with hardware), you'll get plain vanilla Linux. +* You can choose the hardware used with `whirr.hardware-id` to a specific instance type (e.g., m1.large). By default druid.properties, m3.2xlarge (broker, historical, middle manager), m1.xlarge (coordinator, overlord), and m1.small (zookeeper, mysql) are used. +* If you don't choose an image via `whirr.image-id` (image must be compatible with hardware), you'll get plain vanilla Linux. Default druid.properties uses ami-018c9568 (Ubuntu 12.04). * SSH keys (not password protected) must exist for the local user. If they are in the default locations, `${sys:user.home}/.ssh/id_rsa` and `${sys:user.home}/.ssh/id_rsa.pub`, Whirr will find them. Otherwise, you'll have to specify them with `whirr.private-key-file` and `whirr.public-key-file`. * Be sure to specify the absolute path of the Druid realtime spec file `realtime.spec` in `whirr.druid.realtime.spec.path`. -* Two Druid cluster templates (see `whirr.instance-templates`) are provided: a small cluster running on a single EC2 instance, and a larger cluster running on multiple instances. The first is a good test case to start with. +* Also make sure to specify the correct S3 bucket. Otherwise the cluster won't be able to process tasks. +* Two Druid cluster templates (see `whirr.instance-templates`) are provided: a small cluster running on a single EC2 instance, and a larger cluster running on multiple instances. The following AWS information must be set in `druid.properties`, as environment variables, or in the file `$WHIRR_HOME/conf/credentials`: @@ -52,6 +54,8 @@ The following AWS information must be set in `druid.properties`, as environment How to get the IDENTITY and CREDENTIAL keys is discussed above. +In order to configure each node, you can edit `services/druid/src/main/resources/functions/start_druid.sh` for JVM configuration and `services/druid/src/main/resources/functions/configure_[NODE_NAME].sh` for specific node configuration. For more information on configuration, read the Druid documentations about it (http://druid.io/docs/0.6.116/Configuration.html). + ### Start a Test Cluster With Whirr Run the following command: @@ -77,4 +81,16 @@ Note that Whirr will return an exception if any of the nodes fail to launch, and % $WHIRR_HOME/bin/whirr destroy-cluster --config $WHIRR_HOME/recipes/druid.properties ``` +### Testing the Cluster +Now you can run an indexing task and a simple query to see if all the nodes have launched correctly. We are going to use a Wikipedia example again. For a realtime indexing task, run the following command: +```bash +curl -X 'POST' -H 'Content-Type:application/json' -d @#{YOUR_DRUID_DIRECTORY}/examples/indexing/wikipedia_realtime_task.json #{OVERLORD_PUBLIC_IP_ADDR}:#{PORT}/druid/indexer/v1/task +``` +Issuing the request should return a task ID. + +To check the state of the overlord, open up your browser and go to `#{OVERLORD_PUBLIC_IP_ADDR}:#{PORT}/console.html`. + +Next, go to `#{COORDINATOR_PUBLIC_IP_ADDR}:#{PORT}`. Click "View Information about the Cluster"->"Full Cluster View." You should now see the information about servers and segments. If the cluster runs correctly, Segment dimensions and Segment binaryVersion fields should be filled up. Allow few minutes for the segments to be processed. + +Now you should be able to query the data using broker's public IP address. \ No newline at end of file diff --git a/examples/bin/examples/indexing/wikipedia_realtime_task.json b/examples/bin/examples/indexing/wikipedia_realtime_task.json index 3365c50368c..0068176a1a7 100644 --- a/examples/bin/examples/indexing/wikipedia_realtime_task.json +++ b/examples/bin/examples/indexing/wikipedia_realtime_task.json @@ -66,6 +66,6 @@ "timeDimension": "timestamp", "timeFormat": "iso" }, - "windowPeriod": "PT10m", - "segmentGranularity": "hour" + "windowPeriod": "PT0m", + "segmentGranularity": "minute" } \ No newline at end of file diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index 2381c13d282..c67d44a64ab 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -83,7 +83,7 @@ public class CliBroker extends ServerRunnable binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); + JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); // TODO: delete broker JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index 6d4152b9226..577c5bf68e6 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -81,7 +81,7 @@ public class CliHistorical extends ServerRunnable LifecycleModule.register(binder, Server.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); + JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); //TODO: delete historical JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); MetricsModule.register(binder, CacheMonitor.class); } From 11f07b5fbfa9c204abbef074595c96812eaa5634 Mon Sep 17 00:00:00 2001 From: jisookim0513 Date: Thu, 5 Jun 2014 17:51:58 -0700 Subject: [PATCH 10/39] revert changes to wikipedia_realtime_task.json, CliBroker.java, CliHistorical.java --- docs/content/Booting-a-production-cluster.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/Booting-a-production-cluster.md b/docs/content/Booting-a-production-cluster.md index c97e25cfdee..77c22ba5f63 100644 --- a/docs/content/Booting-a-production-cluster.md +++ b/docs/content/Booting-a-production-cluster.md @@ -12,7 +12,7 @@ You can provision individual servers, loading Druid onto each machine (or buildi [Apache Whirr](http://whirr.apache.org/) is a set of libraries for launching cloud services. For Druid, Whirr serves as an easy way to launch a cluster in Amazon AWS by using simple commands and configuration files (called *recipes*). -**NOTE:** Whirr will install Druid 0.6.115. Also, it doesn't work with JDK1.7.0_55. JDK1.7.0_45 recommended. +**NOTE:** Whirr will install Druid 0.6.121. Also, it doesn't work with JDK1.7.0_55. JDK1.7.0_45 recommended. You'll need an AWS account, S3 Bucket and an EC2 key pair from that account so that Whirr can connect to the cloud via the EC2 API. If you haven't generated a key pair, see the [AWS documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) or see this [Whirr FAQ](http://whirr.apache.org/faq.html#how-do-i-find-my-cloud-credentials). From 5df6e07aeef0f3112f2a00fa60c644977d3b9581 Mon Sep 17 00:00:00 2001 From: jisookim0513 Date: Thu, 5 Jun 2014 17:55:52 -0700 Subject: [PATCH 11/39] revert changes to wikipedia_realtime_task.json, CliBroker.java, CliHistorical.java --- examples/bin/examples/indexing/wikipedia_realtime_task.json | 4 ++-- services/src/main/java/io/druid/cli/CliBroker.java | 2 +- services/src/main/java/io/druid/cli/CliHistorical.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/bin/examples/indexing/wikipedia_realtime_task.json b/examples/bin/examples/indexing/wikipedia_realtime_task.json index 0068176a1a7..3365c50368c 100644 --- a/examples/bin/examples/indexing/wikipedia_realtime_task.json +++ b/examples/bin/examples/indexing/wikipedia_realtime_task.json @@ -66,6 +66,6 @@ "timeDimension": "timestamp", "timeFormat": "iso" }, - "windowPeriod": "PT0m", - "segmentGranularity": "minute" + "windowPeriod": "PT10m", + "segmentGranularity": "hour" } \ No newline at end of file diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index c67d44a64ab..2381c13d282 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -83,7 +83,7 @@ public class CliBroker extends ServerRunnable binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); // TODO: delete broker + JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index 577c5bf68e6..6d4152b9226 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -81,7 +81,7 @@ public class CliHistorical extends ServerRunnable LifecycleModule.register(binder, Server.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); //TODO: delete historical + JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); MetricsModule.register(binder, CacheMonitor.class); } From 55f1e2e2a75ae8561b649548d014d5ec51600d5e Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 6 Jun 2014 14:19:55 -0700 Subject: [PATCH 12/39] add dependencies for validator error --- pom.xml | 19 +++++++++++++++---- server/pom.xml | 19 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 2116d7f617f..30d6fdd1989 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,8 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid druid @@ -203,17 +204,17 @@ com.google.inject guice - 4.0-beta + 4.0-beta4 com.google.inject.extensions guice-servlet - 4.0-beta + 4.0-beta4 com.google.inject.extensions guice-multibindings - 4.0-beta + 4.0-beta4 com.ibm.icu @@ -280,6 +281,16 @@ javax.inject 1 + + javax.el + javax.el-api + 2.2.4 + + + org.glassfish.web + javax.el + 2.2.4 + com.jamesmurty.utils java-xmlbuilder diff --git a/server/pom.xml b/server/pom.xml index 83635cdc29f..ee1974b000c 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -18,7 +18,8 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid druid-server @@ -57,6 +58,14 @@ javax.inject javax.inject + + javax.el + javax.el-api + + + org.glassfish.web + javax.el + com.amazonaws aws-java-sdk @@ -130,12 +139,12 @@ jetty-servlets - com.ircclouds.irc - irc-api + com.ircclouds.irc + irc-api - com.maxmind.geoip2 - geoip2 + com.maxmind.geoip2 + geoip2 From d3f2faef92c30a2131782f348678dffeb236c081 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 9 Jun 2014 15:32:22 -0700 Subject: [PATCH 13/39] address code review --- pom.xml | 5 ----- server/pom.xml | 4 ---- 2 files changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index 30d6fdd1989..447fa65c084 100644 --- a/pom.xml +++ b/pom.xml @@ -281,11 +281,6 @@ javax.inject 1 - - javax.el - javax.el-api - 2.2.4 - org.glassfish.web javax.el diff --git a/server/pom.xml b/server/pom.xml index ee1974b000c..f01e920cb77 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -58,10 +58,6 @@ javax.inject javax.inject - - javax.el - javax.el-api - org.glassfish.web javax.el From eb5034ce7f0665f8eb1c9cf2e7430572c317bc73 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 9 Jun 2014 15:33:24 -0700 Subject: [PATCH 14/39] update dep version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 447fa65c084..c513b9bd387 100644 --- a/pom.xml +++ b/pom.xml @@ -284,7 +284,7 @@ org.glassfish.web javax.el - 2.2.4 + 3.0.0 com.jamesmurty.utils From 5e48f3907ba198678e47173d75138affe98cc3cc Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 9 Jun 2014 16:24:12 -0700 Subject: [PATCH 15/39] fix index task new schema defaults --- .../druid/indexing/common/task/IndexTask.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index 32565f1d51a..a71d16c54f6 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -35,7 +34,6 @@ import com.google.common.primitives.Ints; import com.metamx.common.ISE; import com.metamx.common.guava.Comparators; import com.metamx.common.logger.Logger; -import com.metamx.common.parsers.TimestampParser; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; @@ -46,15 +44,14 @@ import io.druid.indexing.common.TaskToolbox; import io.druid.indexing.common.index.YeOldePlumberSchool; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.indexing.DataSchema; -import io.druid.segment.indexing.IngestionSpec; -import io.druid.segment.indexing.TuningConfig; import io.druid.segment.indexing.IOConfig; +import io.druid.segment.indexing.IngestionSpec; import io.druid.segment.indexing.RealtimeTuningConfig; +import io.druid.segment.indexing.TuningConfig; import io.druid.segment.indexing.granularity.GranularitySpec; import io.druid.segment.loading.DataSegmentPusher; import io.druid.segment.realtime.FireDepartmentMetrics; import io.druid.segment.realtime.plumber.Plumber; -import io.druid.segment.realtime.plumber.Sink; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; import io.druid.timeline.partition.ShardSpec; @@ -528,6 +525,9 @@ public class IndexTask extends AbstractFixedIntervalTask @JsonTypeName("index") public static class IndexTuningConfig implements TuningConfig { + private static final int DEFAULT_TARGET_PARTITION_SIZE = 5000000; + private static final int DEFAULT_ROW_FLUSH_BOUNDARY = 500000; + private final int targetPartitionSize; private final int rowFlushBoundary; @@ -537,8 +537,8 @@ public class IndexTask extends AbstractFixedIntervalTask @JsonProperty("rowFlushBoundary") int rowFlushBoundary ) { - this.targetPartitionSize = targetPartitionSize; - this.rowFlushBoundary = rowFlushBoundary; + this.targetPartitionSize = targetPartitionSize == 0 ? DEFAULT_TARGET_PARTITION_SIZE : targetPartitionSize; + this.rowFlushBoundary = rowFlushBoundary == 0 ? DEFAULT_ROW_FLUSH_BOUNDARY : rowFlushBoundary; } @JsonProperty @@ -553,12 +553,4 @@ public class IndexTask extends AbstractFixedIntervalTask return rowFlushBoundary; } } - - - public static void main(String[] args) - { - Function parser = TimestampParser.createTimestampParser("millis"); - parser.apply("1401266370985"); - - } } From ad7db018d5c87a0a4d5b7cbe4cc31dd0b2037526 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 9 Jun 2014 16:28:47 -0700 Subject: [PATCH 16/39] add default tuning config for index task --- .../src/main/java/io/druid/indexing/common/task/IndexTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index a71d16c54f6..182a1569903 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -477,7 +477,7 @@ public class IndexTask extends AbstractFixedIntervalTask this.dataSchema = dataSchema; this.ioConfig = ioConfig; - this.tuningConfig = tuningConfig; + this.tuningConfig = tuningConfig == null ? new IndexTuningConfig(0, 0) : tuningConfig; } @Override From 21908b507784a02576fd64932dc6ac92ec5d7b4c Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Tue, 10 Jun 2014 17:42:30 +0530 Subject: [PATCH 17/39] hdfs data segment killer implement hdfs data segment killer --- .../storage/hdfs/HdfsDataSegmentKiller.java | 76 +++++++++++++++++++ .../storage/hdfs/HdfsStorageDruidModule.java | 1 + 2 files changed, 77 insertions(+) create mode 100644 hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java diff --git a/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java new file mode 100644 index 00000000000..4d4ddf70766 --- /dev/null +++ b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java @@ -0,0 +1,76 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.storage.hdfs; + +import com.google.inject.Inject; +import io.druid.segment.loading.DataSegmentKiller; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.timeline.DataSegment; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +public class HdfsDataSegmentKiller implements DataSegmentKiller +{ + private final Configuration config; + + @Inject + public HdfsDataSegmentKiller(final Configuration config) + { + this.config = config; + } + + @Override + public void kill(DataSegment segment) throws SegmentLoadingException + { + final Path path = getPath(segment); + final FileSystem fs = checkPathAndGetFilesystem(path); + try { + fs.delete(path, true); + } + catch (IOException e) { + throw new SegmentLoadingException(e, "Unable to kill segment"); + } + } + + private Path getPath(DataSegment segment) + { + return new Path(String.valueOf(segment.getLoadSpec().get("path"))); + } + + private FileSystem checkPathAndGetFilesystem(Path path) throws SegmentLoadingException + { + FileSystem fs; + try { + fs = path.getFileSystem(config); + + if (!fs.exists(path)) { + throw new SegmentLoadingException("Path[%s] doesn't exist.", path); + } + + return fs; + } + catch (IOException e) { + throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", path); + } + } +} diff --git a/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java index e2fb1475742..0ebb477a593 100644 --- a/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java +++ b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java @@ -55,6 +55,7 @@ public class HdfsStorageDruidModule implements DruidModule { Binders.dataSegmentPullerBinder(binder).addBinding("hdfs").to(HdfsDataSegmentPuller.class).in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding("hdfs").to(HdfsDataSegmentPusher.class).in(LazySingleton.class); + Binders.dataSegmentKillerBinder(binder).addBinding("hdfs").to(HdfsDataSegmentKiller.class).in(LazySingleton.class); final Configuration conf = new Configuration(); if (props != null) { From f3e60795d0ac9e8b736a2b6a465d1ca76d0c2e65 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Tue, 10 Jun 2014 19:25:08 +0530 Subject: [PATCH 18/39] delete the parent directory --- .../java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java index 4d4ddf70766..bc62fbaa46c 100644 --- a/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java +++ b/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java @@ -45,7 +45,12 @@ public class HdfsDataSegmentKiller implements DataSegmentKiller final Path path = getPath(segment); final FileSystem fs = checkPathAndGetFilesystem(path); try { - fs.delete(path, true); + if (path.getName().endsWith(".zip")) { + // delete the parent directory containing the zip file and the descriptor + fs.delete(path.getParent(), true); + } else { + throw new SegmentLoadingException("Unknown file type[%s]", path); + } } catch (IOException e) { throw new SegmentLoadingException(e, "Unable to kill segment"); From d6f38827db453f31dd1e082e31dbcaade23b085a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 8 Apr 2014 17:13:22 -0700 Subject: [PATCH 19/39] initial query cancellation commit --- .../DruidDefaultSerializersModule.java | 1 + .../query/ChainedExecutionQueryRunner.java | 131 ++++++++-------- .../query/QueryInterruptedException.java | 37 +++++ .../java/io/druid/query/QueryWatcher.java | 27 ++++ .../search/SearchQueryRunnerFactory.java | 8 +- .../select/SelectQueryRunnerFactory.java | 18 ++- .../TimeBoundaryQueryRunnerFactory.java | 11 +- .../TimeseriesQueryRunnerFactory.java | 22 ++- .../query/topn/TopNQueryRunnerFactory.java | 8 +- .../ChainedExecutionQueryRunnerTest.java | 148 ++++++++++++++++++ .../java/io/druid/query/TestQueryRunners.java | 30 +++- .../query/search/SearchQueryRunnerTest.java | 12 +- .../TimeBoundaryQueryRunnerTest.java | 12 +- .../druid/query/topn/TopNQueryRunnerTest.java | 23 ++- .../druid/guice/QueryRunnerFactoryModule.java | 8 + .../java/io/druid/server/QueryManager.java | 48 ++++++ .../java/io/druid/server/QueryResource.java | 21 ++- 17 files changed, 481 insertions(+), 84 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/QueryInterruptedException.java create mode 100644 processing/src/main/java/io/druid/query/QueryWatcher.java create mode 100644 processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java create mode 100644 server/src/main/java/io/druid/server/QueryManager.java diff --git a/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java index 38c3bc135ef..068bfecd963 100644 --- a/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java +++ b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java @@ -32,6 +32,7 @@ import com.google.common.base.Throwables; import com.metamx.common.Granularity; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Yielder; import org.joda.time.DateTimeZone; import java.io.IOException; diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index 31eac12702e..6c12d37669b 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -25,6 +25,10 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.ISE; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.MergeIterable; @@ -35,11 +39,8 @@ import com.metamx.common.logger.Logger; import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; /** * A QueryRunner that combines a list of other QueryRunners and executes them in parallel on an executor. @@ -59,27 +60,33 @@ public class ChainedExecutionQueryRunner implements QueryRunner private static final Logger log = new Logger(ChainedExecutionQueryRunner.class); private final Iterable> queryables; - private final ExecutorService exec; + private final ListeningExecutorService exec; private final Ordering ordering; + private final QueryWatcher queryWatcher; public ChainedExecutionQueryRunner( ExecutorService exec, Ordering ordering, + QueryWatcher queryWatcher, QueryRunner... queryables ) { - this(exec, ordering, Arrays.asList(queryables)); + this(exec, ordering, queryWatcher, Arrays.asList(queryables)); } public ChainedExecutionQueryRunner( ExecutorService exec, Ordering ordering, + QueryWatcher queryWatcher, Iterable> queryables ) { - this.exec = exec; + // listeningDecorator will leave PrioritizedExecutorService unchanged, + // since it already implements ListeningExecutorService + this.exec = MoreExecutors.listeningDecorator(exec); this.ordering = ordering; this.queryables = Iterables.unmodifiableIterable(Iterables.filter(queryables, Predicates.notNull())); + this.queryWatcher = queryWatcher; } @Override @@ -94,71 +101,67 @@ public class ChainedExecutionQueryRunner implements QueryRunner public Iterator make() { // Make it a List<> to materialize all of the values (so that it will submit everything to the executor) - List>> futures = Lists.newArrayList( - Iterables.transform( - queryables, - new Function, Future>>() - { - @Override - public Future> apply(final QueryRunner input) - { - return exec.submit( - new AbstractPrioritizedCallable>(priority) - { - @Override - public List call() throws Exception - { - try { - if (input == null) { - throw new ISE("Input is null?! How is this possible?!"); - } + ListenableFuture>> futures = Futures.allAsList( + Lists.newArrayList( + Iterables.transform( + queryables, + new Function, ListenableFuture>>() + { + @Override + public ListenableFuture> apply(final QueryRunner input) + { + return exec.submit( + new AbstractPrioritizedCallable>(priority) + { + @Override + public Iterable call() throws Exception + { + try { + if (input == null) { + throw new ISE("Input is null?! How is this possible?!"); + } - Sequence result = input.run(query); - if (result == null) { - throw new ISE("Got a null result! Segments are missing!"); - } + Sequence result = input.run(query); + if (result == null) { + throw new ISE("Got a null result! Segments are missing!"); + } - List retVal = Sequences.toList(result, Lists.newArrayList()); - if (retVal == null) { - throw new ISE("Got a null list of results! WTF?!"); - } + List retVal = Sequences.toList(result, Lists.newArrayList()); + if (retVal == null) { + throw new ISE("Got a null list of results! WTF?!"); + } - return retVal; + return retVal; + } + catch (Exception e) { + log.error(e, "Exception with one of the sequences!"); + throw Throwables.propagate(e); + } + } } - catch (Exception e) { - log.error(e, "Exception with one of the sequences!"); - throw Throwables.propagate(e); - } - } - } - ); - } - } + ); + } + } + ) ) ); - return new MergeIterable( - ordering.nullsFirst(), - Iterables.transform( - futures, - new Function>, Iterable>() - { - @Override - public Iterable apply(Future> input) - { - try { - return input.get(); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - catch (ExecutionException e) { - throw new RuntimeException(e); - } - } - } - ) - ).iterator(); + queryWatcher.registerQuery(query, futures); + + try { + return new MergeIterable<>( + ordering.nullsFirst(), + futures.get() + ).iterator(); + } + catch (InterruptedException e) { + log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId()); + futures.cancel(true); + throw new QueryInterruptedException(e); + } + catch (ExecutionException e) { + throw Throwables.propagate(e.getCause()); + } } @Override diff --git a/processing/src/main/java/io/druid/query/QueryInterruptedException.java b/processing/src/main/java/io/druid/query/QueryInterruptedException.java new file mode 100644 index 00000000000..7b889ef5f2d --- /dev/null +++ b/processing/src/main/java/io/druid/query/QueryInterruptedException.java @@ -0,0 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query; + +public class QueryInterruptedException extends RuntimeException +{ + public QueryInterruptedException() { + super(); + } + + public QueryInterruptedException(String message) + { + super(message); + } + + public QueryInterruptedException(Throwable cause) + { + super(cause); + } +} diff --git a/processing/src/main/java/io/druid/query/QueryWatcher.java b/processing/src/main/java/io/druid/query/QueryWatcher.java new file mode 100644 index 00000000000..0a76a54f23a --- /dev/null +++ b/processing/src/main/java/io/druid/query/QueryWatcher.java @@ -0,0 +1,27 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface QueryWatcher +{ + public void registerQuery(Query query, ListenableFuture future); +} diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java index 0c0fdab980d..2cd868e45d8 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java @@ -24,6 +24,7 @@ import io.druid.query.ChainedExecutionQueryRunner; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.search.search.SearchQuery; import io.druid.segment.Segment; @@ -35,13 +36,16 @@ import java.util.concurrent.ExecutorService; public class SearchQueryRunnerFactory implements QueryRunnerFactory, SearchQuery> { private final SearchQueryQueryToolChest toolChest; + private final QueryWatcher queryWatcher; @Inject public SearchQueryRunnerFactory( - SearchQueryQueryToolChest toolChest + SearchQueryQueryToolChest toolChest, + QueryWatcher queryWatcher ) { this.toolChest = toolChest; + this.queryWatcher = queryWatcher; } @Override @@ -56,7 +60,7 @@ public class SearchQueryRunnerFactory implements QueryRunnerFactory>( - queryExecutor, toolChest.getOrdering(), queryRunners + queryExecutor, toolChest.getOrdering(), queryWatcher, queryRunners ); } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java index 6e995b15f44..a1fa77dabd5 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java @@ -20,6 +20,7 @@ package io.druid.query.select; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; @@ -29,6 +30,7 @@ import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.segment.Segment; @@ -43,21 +45,31 @@ public class SelectQueryRunnerFactory { return new SelectQueryRunnerFactory( new SelectQueryQueryToolChest(new QueryConfig(), jsonMapper), - new SelectQueryEngine() + new SelectQueryEngine(), + new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + } + } ); } private final SelectQueryQueryToolChest toolChest; private final SelectQueryEngine engine; + private final QueryWatcher queryWatcher; @Inject public SelectQueryRunnerFactory( SelectQueryQueryToolChest toolChest, - SelectQueryEngine engine + SelectQueryEngine engine, + QueryWatcher queryWatcher ) { this.toolChest = toolChest; this.engine = engine; + this.queryWatcher = queryWatcher; } @Override @@ -72,7 +84,7 @@ public class SelectQueryRunnerFactory ) { return new ChainedExecutionQueryRunner>( - queryExecutor, toolChest.getOrdering(), queryRunners + queryExecutor, toolChest.getOrdering(), queryWatcher, queryRunners ); } diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java index 16e9ae832fa..1f78429ead3 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java @@ -19,6 +19,7 @@ package io.druid.query.timeboundary; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.Sequence; @@ -27,6 +28,7 @@ import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; @@ -40,6 +42,13 @@ public class TimeBoundaryQueryRunnerFactory implements QueryRunnerFactory, TimeBoundaryQuery> { private static final TimeBoundaryQueryQueryToolChest toolChest = new TimeBoundaryQueryQueryToolChest(); + private final QueryWatcher queryWatcher; + + @Inject + public TimeBoundaryQueryRunnerFactory(QueryWatcher queryWatcher) + { + this.queryWatcher = queryWatcher; + } @Override public QueryRunner> createRunner(final Segment segment) @@ -53,7 +62,7 @@ public class TimeBoundaryQueryRunnerFactory ) { return new ChainedExecutionQueryRunner>( - queryExecutor, toolChest.getOrdering(), queryRunners + queryExecutor, toolChest.getOrdering(), queryWatcher, queryRunners ); } diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java index 740b309e37e..726bc20bb43 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java @@ -19,6 +19,7 @@ package io.druid.query.timeseries; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; @@ -28,6 +29,7 @@ import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; @@ -39,25 +41,39 @@ import java.util.concurrent.ExecutorService; public class TimeseriesQueryRunnerFactory implements QueryRunnerFactory, TimeseriesQuery> { + /** + * Use only for testing + * @return + */ public static TimeseriesQueryRunnerFactory create() { return new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(new QueryConfig()), - new TimeseriesQueryEngine() + new TimeseriesQueryEngine(), + new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + } + } ); } private final TimeseriesQueryQueryToolChest toolChest; private final TimeseriesQueryEngine engine; + private final QueryWatcher queryWatcher; @Inject public TimeseriesQueryRunnerFactory( TimeseriesQueryQueryToolChest toolChest, - TimeseriesQueryEngine engine + TimeseriesQueryEngine engine, + QueryWatcher queryWatcher ) { this.toolChest = toolChest; this.engine = engine; + this.queryWatcher = queryWatcher; } @Override @@ -72,7 +88,7 @@ public class TimeseriesQueryRunnerFactory ) { return new ChainedExecutionQueryRunner>( - queryExecutor, toolChest.getOrdering(), queryRunners + queryExecutor, toolChest.getOrdering(), queryWatcher, queryRunners ); } diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/topn/TopNQueryRunnerFactory.java index 044e6a64eb2..524f9ace6a5 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryRunnerFactory.java @@ -30,6 +30,7 @@ import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.segment.Segment; @@ -43,15 +44,18 @@ public class TopNQueryRunnerFactory implements QueryRunnerFactory computationBufferPool; private final TopNQueryQueryToolChest toolchest; + private final QueryWatcher queryWatcher; @Inject public TopNQueryRunnerFactory( @Global StupidPool computationBufferPool, - TopNQueryQueryToolChest toolchest + TopNQueryQueryToolChest toolchest, + QueryWatcher queryWatcher ) { this.computationBufferPool = computationBufferPool; this.toolchest = toolchest; + this.queryWatcher = queryWatcher; } @Override @@ -79,7 +83,7 @@ public class TopNQueryRunnerFactory implements QueryRunnerFactory>( - queryExecutor, toolchest.getOrdering(), queryRunners + queryExecutor, toolchest.getOrdering(), queryWatcher, queryRunners ); } diff --git a/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java new file mode 100644 index 00000000000..445bea9cf53 --- /dev/null +++ b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java @@ -0,0 +1,148 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.util.concurrent.ListenableFuture; +import com.metamx.common.concurrent.ExecutorServiceConfig; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import com.metamx.common.lifecycle.Lifecycle; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class ChainedExecutionQueryRunnerTest +{ + @Test @Ignore + public void testQueryCancellation() throws Exception + { + ExecutorService exec = PrioritizedExecutorService.create( + new Lifecycle(), new ExecutorServiceConfig() + { + @Override + public String getFormatString() + { + return "test"; + } + + @Override + public int getNumThreads() + { + return 2; + } + } + ); + + final CountDownLatch queriesStarted = new CountDownLatch(2); + final CountDownLatch queryIsRegistered = new CountDownLatch(1); + + final Map queries = Maps.newHashMap(); + QueryWatcher watcher = new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + queries.put(query, future); + queryIsRegistered.countDown(); + } + }; + + ChainedExecutionQueryRunner chainedRunner = new ChainedExecutionQueryRunner<>( + exec, + Ordering.natural(), + watcher, + Lists.>newArrayList( + new DyingQueryRunner(1, queriesStarted), + new DyingQueryRunner(2, queriesStarted), + new DyingQueryRunner(3, queriesStarted) + ) + ); + + final Sequence seq = chainedRunner.run( + Druids.newTimeseriesQueryBuilder() + .dataSource("test") + .intervals("2014/2015") + .aggregators(Lists.newArrayList(new CountAggregatorFactory("count"))) + .build() + ); + + Future f = Executors.newFixedThreadPool(1).submit( + new Runnable() + { + @Override + public void run() + { + Sequences.toList(seq, Lists.newArrayList()); + } + } + ); + + // wait for query to register + queryIsRegistered.await(); + queriesStarted.await(); + + // cancel the query + queries.values().iterator().next().cancel(true); + f.get(); + } + + private static class DyingQueryRunner implements QueryRunner + { + private final int id; + private final CountDownLatch latch; + + public DyingQueryRunner(int id, CountDownLatch latch) { + this.id = id; + this.latch = latch; + } + + @Override + public Sequence run(Query query) + { + latch.countDown(); + + int i = 0; + while (i >= 0) { + if(Thread.interrupted()) { + throw new QueryInterruptedException("I got killed"); + } + + // do a lot of work + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new QueryInterruptedException("I got killed"); + } + ++i; + } + return Sequences.simple(Lists.newArrayList(i)); + } + } +} diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index c4767c1c6f9..a858b5e0cdf 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -1,6 +1,7 @@ package io.druid.query; import com.google.common.base.Supplier; +import com.google.common.util.concurrent.ListenableFuture; import io.druid.collections.StupidPool; import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; @@ -40,7 +41,14 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new TopNQueryRunnerFactory(pool, new TopNQueryQueryToolChest(topNConfig)); + QueryRunnerFactory factory = new TopNQueryRunnerFactory(pool, new TopNQueryQueryToolChest(topNConfig), new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() @@ -62,7 +70,14 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig())); + QueryRunnerFactory factory = new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig()), new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() @@ -73,11 +88,18 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new TimeBoundaryQueryRunnerFactory(); + QueryRunnerFactory factory = new TimeBoundaryQueryRunnerFactory(new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() ); } -} \ No newline at end of file +} diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 2e5623b4365..0740333eed5 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -23,10 +23,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.guava.Sequences; import io.druid.query.Druids; +import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.filter.DimFilter; import io.druid.query.search.search.FragmentSearchQuerySpec; @@ -56,7 +59,14 @@ public class SearchQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig())) + new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig()), new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }) ); } diff --git a/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java index 9b92826ac34..de5ac1281b2 100644 --- a/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java @@ -20,10 +20,13 @@ package io.druid.query.timeboundary; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.guava.Sequences; import io.druid.query.Druids; +import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import org.joda.time.DateTime; import org.junit.Assert; @@ -43,7 +46,14 @@ public class TimeBoundaryQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - new TimeBoundaryQueryRunnerFactory() + new TimeBoundaryQueryRunnerFactory(new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }) ); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 22b750faf00..39af4459794 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -23,12 +23,15 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.guava.Sequences; import io.druid.collections.StupidPool; import io.druid.query.BySegmentResultValueClass; import io.druid.query.Druids; +import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.TestQueryRunners; import io.druid.query.aggregation.AggregatorFactory; @@ -68,7 +71,15 @@ public class TopNQueryRunnerTest QueryRunnerTestHelper.makeQueryRunners( new TopNQueryRunnerFactory( TestQueryRunners.getPool(), - new TopNQueryQueryToolChest(new TopNQueryConfig()) + new TopNQueryQueryToolChest(new TopNQueryConfig()), + new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + } ) ) ); @@ -85,7 +96,15 @@ public class TopNQueryRunnerTest } } ), - new TopNQueryQueryToolChest(new TopNQueryConfig()) + new TopNQueryQueryToolChest(new TopNQueryConfig()), + new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + } ) ) ); diff --git a/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java index 6f4dc80b059..6c1bb62cafe 100644 --- a/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java @@ -24,6 +24,7 @@ import com.google.inject.Binder; import com.google.inject.multibindings.MapBinder; import io.druid.query.Query; import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryWatcher; import io.druid.query.groupby.GroupByQuery; import io.druid.query.groupby.GroupByQueryEngine; import io.druid.query.groupby.GroupByQueryRunnerFactory; @@ -39,6 +40,7 @@ import io.druid.query.timeseries.TimeseriesQuery; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNQueryRunnerFactory; +import io.druid.server.QueryManager; import java.util.Map; @@ -62,6 +64,12 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule { super.configure(binder); + binder.bind(QueryWatcher.class) + .to(QueryManager.class) + .in(LazySingleton.class); + binder.bind(QueryManager.class) + .in(LazySingleton.class); + final MapBinder, QueryRunnerFactory> queryFactoryBinder = DruidBinders.queryRunnerFactoryBinder( binder ); diff --git a/server/src/main/java/io/druid/server/QueryManager.java b/server/src/main/java/io/druid/server/QueryManager.java new file mode 100644 index 00000000000..5910aeda535 --- /dev/null +++ b/server/src/main/java/io/druid/server/QueryManager.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.query.Query; +import io.druid.query.QueryWatcher; + +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; + +public class QueryManager implements QueryWatcher +{ + final ConcurrentMap queries; + + public QueryManager() { + this.queries = Maps.newConcurrentMap(); + } + + public void cancelQuery(String id) { + Future future = queries.get(id); + if(future != null) { + future.cancel(true); + } + } + public void registerQuery(Query query, ListenableFuture future) + { + queries.put(query.getId(), future); + } +} diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index 6319a0d4633..e710a6c97c9 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -37,16 +37,20 @@ import io.druid.guice.annotations.Smile; import io.druid.query.DataSourceUtil; import io.druid.query.Query; import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryWatcher; import io.druid.server.log.RequestLogger; import org.joda.time.DateTime; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DELETE; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; @@ -66,6 +70,7 @@ public class QueryResource private final QuerySegmentWalker texasRanger; private final ServiceEmitter emitter; private final RequestLogger requestLogger; + private final QueryManager queryManager; @Inject public QueryResource( @@ -73,7 +78,8 @@ public class QueryResource @Smile ObjectMapper smileMapper, QuerySegmentWalker texasRanger, ServiceEmitter emitter, - RequestLogger requestLogger + RequestLogger requestLogger, + QueryManager queryManager ) { this.jsonMapper = jsonMapper; @@ -81,6 +87,16 @@ public class QueryResource this.texasRanger = texasRanger; this.emitter = emitter; this.requestLogger = requestLogger; + this.queryManager = queryManager; + } + + @DELETE + @Path("{id}") + @Produces("application/json") + public Response getServer(@PathParam("id") String queryId) + { + queryManager.cancelQuery(queryId); + return Response.status(Response.Status.ACCEPTED).build(); } @POST @@ -124,10 +140,13 @@ public class QueryResource resp.setStatus(200); resp.setContentType("application/x-javascript"); + resp.setHeader("X-Druid-Query-Id", query.getId()); out = resp.getOutputStream(); jsonWriter.writeValue(out, results); +// JsonGenerator jgen = jsonWriter.getFactory().createGenerator(out); + long requestTime = System.currentTimeMillis() - start; emitter.emit( From 1be85af32019afd80252ebc2aa746640afa8fb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 9 May 2014 22:51:43 -0700 Subject: [PATCH 20/39] handle query interruption at cursor level --- .../io/druid/segment/QueryableIndexStorageAdapter.java | 7 +++++++ .../incremental/IncrementalIndexStorageAdapter.java | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index a71297b3b89..f624cdfa807 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -27,6 +27,7 @@ import com.google.common.io.Closeables; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import io.druid.granularity.QueryGranularity; +import io.druid.query.QueryInterruptedException; import io.druid.query.filter.Filter; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -224,6 +225,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter @Override public void advance() { + if (Thread.interrupted()) { + throw new QueryInterruptedException(); + } cursorOffset.increment(); } @@ -652,6 +656,9 @@ public class QueryableIndexStorageAdapter implements StorageAdapter @Override public void advance() { + if (Thread.interrupted()) { + throw new QueryInterruptedException(); + } ++currRow; } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 0eddf59ac98..3fe807b2761 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -29,6 +29,7 @@ import com.metamx.collections.spatial.search.Bound; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import io.druid.granularity.QueryGranularity; +import io.druid.query.QueryInterruptedException; import io.druid.query.aggregation.Aggregator; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; @@ -239,6 +240,10 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter Iterators.advance(baseIter, numAdvanced); } + if (Thread.interrupted()) { + throw new QueryInterruptedException(); + } + boolean foundMatched = false; while (baseIter.hasNext()) { currEntry.set(baseIter.next()); From 32f6243be0a765bb59389671132b9fb9d35f66cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 9 May 2014 23:28:42 -0700 Subject: [PATCH 21/39] proper closing of resources in case of query cancellation --- .../timeseries/TimeseriesQueryEngine.java | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java index 9f1d0860aa7..ad290536b30 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java @@ -60,27 +60,29 @@ public class TimeseriesQueryEngine { Aggregator[] aggregators = QueryRunnerHelper.makeAggregators(cursor, aggregatorSpecs); - while (!cursor.isDone()) { - for (Aggregator aggregator : aggregators) { - aggregator.aggregate(); + try { + while (!cursor.isDone()) { + for (Aggregator aggregator : aggregators) { + aggregator.aggregate(); + } + cursor.advance(); } - cursor.advance(); + + TimeseriesResultBuilder bob = new TimeseriesResultBuilder(cursor.getTime()); + + for (Aggregator aggregator : aggregators) { + bob.addMetric(aggregator); + } + + Result retVal = bob.build(); + return retVal; } - - TimeseriesResultBuilder bob = new TimeseriesResultBuilder(cursor.getTime()); - - for (Aggregator aggregator : aggregators) { - bob.addMetric(aggregator); + finally { + // cleanup + for (Aggregator agg : aggregators) { + agg.close(); + } } - - Result retVal = bob.build(); - - // cleanup - for (Aggregator agg : aggregators) { - agg.close(); - } - - return retVal; } } ); From 4f1e1576397e08eb26413b76108d0c03eb6bbefd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 14 May 2014 16:58:41 -0700 Subject: [PATCH 22/39] pass through errors from computes in DirectDruidClient --- .../io/druid/query/QueryInterruptedException.java | 13 ++++++++++++- .../java/io/druid/client/DirectDruidClient.java | 8 +++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/io/druid/query/QueryInterruptedException.java b/processing/src/main/java/io/druid/query/QueryInterruptedException.java index 7b889ef5f2d..00676918dcf 100644 --- a/processing/src/main/java/io/druid/query/QueryInterruptedException.java +++ b/processing/src/main/java/io/druid/query/QueryInterruptedException.java @@ -19,13 +19,17 @@ package io.druid.query; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + public class QueryInterruptedException extends RuntimeException { public QueryInterruptedException() { super(); } - public QueryInterruptedException(String message) + @JsonCreator + public QueryInterruptedException(@JsonProperty("error") String message) { super(message); } @@ -34,4 +38,11 @@ public class QueryInterruptedException extends RuntimeException { super(cause); } + + @JsonProperty("error") + @Override + public String getMessage() + { + return super.getMessage(); + } } diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index ae994d1cda7..c08cd9e2bd4 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -45,6 +45,7 @@ import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; import io.druid.query.BySegmentResultValueClass; import io.druid.query.Query; +import io.druid.query.QueryInterruptedException; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.QueryToolChestWarehouse; @@ -283,7 +284,12 @@ public class DirectDruidClient implements QueryRunner if (jp == null) { try { jp = objectMapper.getFactory().createParser(future.get()); - if (jp.nextToken() != JsonToken.START_ARRAY) { + final JsonToken nextToken = jp.nextToken(); + if (nextToken == JsonToken.START_OBJECT) { + QueryInterruptedException e = jp.getCodec().readValue(jp, QueryInterruptedException.class); + throw e; + } + else if (nextToken != JsonToken.START_ARRAY) { throw new IAE("Next token wasn't a START_ARRAY, was[%s] from url [%s]", jp.getCurrentToken(), url); } else { jp.nextToken(); From 1183c68ce4e2c55529be8bcf7fcd7c087af2fdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 14 May 2014 16:59:01 -0700 Subject: [PATCH 23/39] formatting --- .../main/java/io/druid/segment/ReferenceCountingSequence.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java b/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java index 402e63dc31d..4c5cbfc3d3d 100644 --- a/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java +++ b/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java @@ -48,4 +48,4 @@ public class ReferenceCountingSequence extends YieldingSequenceBase final Closeable closeable = segment.increment(); return new ResourceClosingYielder(baseSequence.toYielder(initValue, accumulator), closeable); } -} \ No newline at end of file +} From a56a655eae2f4dfb5492f278d5bb0078ec3293bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 14 May 2014 17:00:18 -0700 Subject: [PATCH 24/39] proper query exceptions and add support for query timeout --- .../query/ChainedExecutionQueryRunner.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index 6c12d37669b..c7ed29f1ddf 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -39,8 +39,11 @@ import com.metamx.common.logger.Logger; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * A QueryRunner that combines a list of other QueryRunners and executes them in parallel on an executor. @@ -149,15 +152,26 @@ public class ChainedExecutionQueryRunner implements QueryRunner queryWatcher.registerQuery(query, futures); try { + final Number timeout = query.getContextValue("timeout", (Number)null); return new MergeIterable<>( ordering.nullsFirst(), - futures.get() + timeout == null ? + futures.get() : + futures.get(timeout.longValue(), TimeUnit.MILLISECONDS) ).iterator(); } catch (InterruptedException e) { log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId()); futures.cancel(true); - throw new QueryInterruptedException(e); + throw new QueryInterruptedException("Query interrupted"); + } + catch(CancellationException e) { + log.warn(e, "Query cancelled, query id [%s]", query.getId()); + throw new QueryInterruptedException("Query cancelled"); + } + catch(TimeoutException e) { + log.warn(e, "Query timeout, query id [%s]", query.getId()); + throw new QueryInterruptedException("Query timeout"); } catch (ExecutionException e) { throw Throwables.propagate(e.getCause()); From def62c74f8e60f799b7118d26554e5f340ca9cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 14 May 2014 17:01:18 -0700 Subject: [PATCH 25/39] properly remove completed queries from query manager --- .../java/io/druid/server/QueryManager.java | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/io/druid/server/QueryManager.java b/server/src/main/java/io/druid/server/QueryManager.java index 5910aeda535..84b947414dd 100644 --- a/server/src/main/java/io/druid/server/QueryManager.java +++ b/server/src/main/java/io/druid/server/QueryManager.java @@ -19,30 +19,50 @@ package io.druid.server; -import com.google.common.collect.Maps; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.SetMultimap; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.druid.query.Query; import io.druid.query.QueryWatcher; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; +import java.util.Set; public class QueryManager implements QueryWatcher { - final ConcurrentMap queries; + final SetMultimap queries; - public QueryManager() { - this.queries = Maps.newConcurrentMap(); - } - - public void cancelQuery(String id) { - Future future = queries.get(id); - if(future != null) { - future.cancel(true); - } - } - public void registerQuery(Query query, ListenableFuture future) + public QueryManager() { - queries.put(query.getId(), future); + this.queries = Multimaps.synchronizedSetMultimap( + HashMultimap.create() + ); + } + + public boolean cancelQuery(String id) { + Set futures = queries.removeAll(id); + boolean success = true; + for (ListenableFuture future : futures) { + success = success && future.cancel(true); + } + return success; + } + + public void registerQuery(Query query, final ListenableFuture future) + { + final String id = query.getId(); + queries.put(id, future); + future.addListener( + new Runnable() + { + @Override + public void run() + { + queries.remove(id, future); + } + }, + MoreExecutors.sameThreadExecutor() + ); } } From d2c729adec3b9b82705646c6f006ad55b8860a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 12 May 2014 15:07:49 -0700 Subject: [PATCH 26/39] return proper json errors to propagate query cancellation and timeout --- .../io/druid/jackson/DefaultObjectMapper.java | 13 +- .../DruidDefaultSerializersModule.java | 26 ++- .../java/io/druid/server/QueryResource.java | 154 +++++++++++------- 3 files changed, 127 insertions(+), 66 deletions(-) diff --git a/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java index 952b506bd95..e55e2299d2b 100644 --- a/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java +++ b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java @@ -32,7 +32,12 @@ public class DefaultObjectMapper extends ObjectMapper { public DefaultObjectMapper() { - this(null); + this((JsonFactory)null); + } + + public DefaultObjectMapper(DefaultObjectMapper mapper) + { + super(mapper); } public DefaultObjectMapper(JsonFactory factory) @@ -52,4 +57,10 @@ public class DefaultObjectMapper extends ObjectMapper configure(MapperFeature.AUTO_DETECT_SETTERS, false); configure(SerializationFeature.INDENT_OUTPUT, false); } + + @Override + public ObjectMapper copy() + { + return new DefaultObjectMapper(this); + } } diff --git a/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java index 068bfecd963..6184221a1db 100644 --- a/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java +++ b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java @@ -105,7 +105,7 @@ public class DruidDefaultSerializersModule extends SimpleModule jgen.writeStartArray(); value.accumulate( null, - new Accumulator() + new Accumulator() { @Override public Object accumulate(Object o, Object o1) @@ -116,7 +116,7 @@ public class DruidDefaultSerializersModule extends SimpleModule catch (IOException e) { throw Throwables.propagate(e); } - return o; + return null; } } ); @@ -124,6 +124,28 @@ public class DruidDefaultSerializersModule extends SimpleModule } } ); + addSerializer( + Yielder.class, + new JsonSerializer() + { + @Override + public void serialize(Yielder yielder, final JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException + { + jgen.writeStartArray(); + try { + while (!yielder.isDone()) { + final Object o = yielder.get(); + jgen.writeObject(o); + yielder = yielder.next(null); + } + } finally { + yielder.close(); + } + jgen.writeEndArray(); + } + } + ); addSerializer(ByteOrder.class, ToStringSerializer.instance); addDeserializer( ByteOrder.class, diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index e710a6c97c9..1e6ea06607f 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -19,16 +19,23 @@ package io.druid.server; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.api.client.repackaged.com.google.common.base.Throwables; import com.google.common.base.Charsets; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import com.google.inject.Inject; +import com.metamx.common.guava.Accumulator; +import com.metamx.common.guava.Accumulators; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import com.metamx.common.guava.Yielder; +import com.metamx.common.guava.YieldingAccumulator; +import com.metamx.common.guava.YieldingAccumulators; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; @@ -36,21 +43,25 @@ import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; import io.druid.query.DataSourceUtil; import io.druid.query.Query; +import io.druid.query.QueryInterruptedException; import io.druid.query.QuerySegmentWalker; -import io.druid.query.QueryWatcher; import io.druid.server.log.RequestLogger; import org.joda.time.DateTime; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; @@ -64,6 +75,8 @@ public class QueryResource private static final EmittingLogger log = new EmittingLogger(QueryResource.class); private static final Charset UTF8 = Charset.forName("UTF-8"); private static final Joiner COMMA_JOIN = Joiner.on(","); + public static final String APPLICATION_SMILE = "application/smile"; + public static final String APPLICATION_JSON = "application/json"; private final ObjectMapper jsonMapper; private final ObjectMapper smileMapper; @@ -82,8 +95,12 @@ public class QueryResource QueryManager queryManager ) { - this.jsonMapper = jsonMapper; - this.smileMapper = smileMapper; + this.jsonMapper = jsonMapper.copy(); + this.jsonMapper.getFactory().configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); + + this.smileMapper = smileMapper.copy(); + this.smileMapper.getFactory().configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); + this.texasRanger = texasRanger; this.emitter = emitter; this.requestLogger = requestLogger; @@ -97,13 +114,13 @@ public class QueryResource { queryManager.cancelQuery(queryId); return Response.status(Response.Status.ACCEPTED).build(); + } @POST - @Produces("application/json") - public void doPost( + public Response doPost( @Context HttpServletRequest req, - @Context HttpServletResponse resp + @Context final HttpServletResponse resp ) throws ServletException, IOException { final long start = System.currentTimeMillis(); @@ -111,13 +128,12 @@ public class QueryResource byte[] requestQuery = null; String queryId; - final boolean isSmile = "application/smile".equals(req.getContentType()); + final boolean isSmile = APPLICATION_SMILE.equals(req.getContentType()); ObjectMapper objectMapper = isSmile ? smileMapper : jsonMapper; - ObjectWriter jsonWriter = req.getParameter("pretty") == null + final ObjectWriter jsonWriter = req.getParameter("pretty") == null ? objectMapper.writer() : objectMapper.writerWithDefaultPrettyPrinter(); - OutputStream out = null; try { requestQuery = ByteStreams.toByteArray(req.getInputStream()); @@ -132,48 +148,70 @@ public class QueryResource log.debug("Got query [%s]", query); } - Sequence results = query.run(texasRanger); + Sequence results = query.run(texasRanger); if (results == null) { results = Sequences.empty(); } - resp.setStatus(200); - resp.setContentType("application/x-javascript"); - resp.setHeader("X-Druid-Query-Id", query.getId()); - - out = resp.getOutputStream(); - jsonWriter.writeValue(out, results); - -// JsonGenerator jgen = jsonWriter.getFactory().createGenerator(out); - - long requestTime = System.currentTimeMillis() - start; - - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) - .setUser4(query.getType()) - .setUser5(COMMA_JOIN.join(query.getIntervals())) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(req.getRemoteAddr()) - .setUser8(queryId) - .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()) - .build("request/time", requestTime) - ); - - requestLogger.log( - new RequestLogLine( - new DateTime(), - req.getRemoteAddr(), - query, - new QueryStats( - ImmutableMap.of( - "request/time", requestTime, - "success", true - ) - ) + try ( + final Yielder yielder = results.toYielder( + null, + new YieldingAccumulator() + { + @Override + public Object accumulate(Object accumulated, Object in) + { + yield(); + return in; + } + } ) - ); + ) { + long requestTime = System.currentTimeMillis() - start; + + emitter.emit( + new ServiceMetricEvent.Builder() + .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) + .setUser4(query.getType()) + .setUser5(COMMA_JOIN.join(query.getIntervals())) + .setUser6(String.valueOf(query.hasFilters())) + .setUser7(req.getRemoteAddr()) + .setUser8(queryId) + .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()) + .build("request/time", requestTime) + ); + + requestLogger.log( + new RequestLogLine( + new DateTime(), + req.getRemoteAddr(), + query, + new QueryStats( + ImmutableMap.of( + "request/time", requestTime, + "success", true + ) + ) + ) + ); + + return Response + .ok( + new StreamingOutput() + { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException + { + jsonWriter.writeValue(outputStream, yielder); + outputStream.close(); + } + }, + isSmile ? APPLICATION_JSON : APPLICATION_SMILE + ) + .header("X-Druid-Query-Id", queryId) + .build(); + } } catch (Exception e) { final String queryString = @@ -183,20 +221,6 @@ public class QueryResource log.warn(e, "Exception occurred on request [%s]", queryString); - if (!resp.isCommitted()) { - resp.setStatus(500); - resp.resetBuffer(); - - if (out == null) { - out = resp.getOutputStream(); - } - - out.write((e.getMessage() == null) ? "Exception null".getBytes(UTF8) : e.getMessage().getBytes(UTF8)); - out.write("\n".getBytes(UTF8)); - } - - resp.flushBuffer(); - try { requestLogger.log( new RequestLogLine( @@ -216,10 +240,14 @@ public class QueryResource .addData("query", queryString) .addData("peer", req.getRemoteAddr()) .emit(); - } - finally { - resp.flushBuffer(); - Closeables.closeQuietly(out); + + return Response.serverError().entity( + jsonWriter.writeValueAsString( + ImmutableMap.of( + "error", (e.getMessage() == null) ? "null Exception" : e.getMessage() + ) + ) + ).build(); } } } From d01f272a7ab71ab7ba565493404f8330af3e774b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 2 Jun 2014 17:40:35 -0700 Subject: [PATCH 27/39] forward cancellation in direct druid client --- pom.xml | 2 +- .../io/druid/client/BrokerServerView.java | 8 +- .../io/druid/client/DirectDruidClient.java | 46 ++++++++-- .../java/io/druid/server/QueryResource.java | 1 - .../druid/client/DirectDruidClientTest.java | 90 +++++++++++++++++-- 5 files changed, 127 insertions(+), 20 deletions(-) diff --git a/pom.xml b/pom.xml index 2116d7f617f..2a3af54886e 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ com.metamx http-client - 0.9.5 + 0.9.6 com.metamx diff --git a/server/src/main/java/io/druid/client/BrokerServerView.java b/server/src/main/java/io/druid/client/BrokerServerView.java index 57663154156..0070622ac85 100644 --- a/server/src/main/java/io/druid/client/BrokerServerView.java +++ b/server/src/main/java/io/druid/client/BrokerServerView.java @@ -32,10 +32,9 @@ import io.druid.client.selector.TierSelectorStrategy; import io.druid.concurrent.Execs; import io.druid.guice.annotations.Client; import io.druid.query.DataSource; -import io.druid.query.QueryDataSource; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChestWarehouse; -import io.druid.query.TableDataSource; +import io.druid.query.QueryWatcher; import io.druid.server.coordination.DruidServerMetadata; import io.druid.timeline.DataSegment; import io.druid.timeline.VersionedIntervalTimeline; @@ -60,6 +59,7 @@ public class BrokerServerView implements TimelineServerView private final Map> timelines; private final QueryToolChestWarehouse warehouse; + private final QueryWatcher queryWatcher; private final ObjectMapper smileMapper; private final HttpClient httpClient; private final ServerInventoryView baseView; @@ -68,6 +68,7 @@ public class BrokerServerView implements TimelineServerView @Inject public BrokerServerView( QueryToolChestWarehouse warehouse, + QueryWatcher queryWatcher, ObjectMapper smileMapper, @Client HttpClient httpClient, ServerInventoryView baseView, @@ -75,6 +76,7 @@ public class BrokerServerView implements TimelineServerView ) { this.warehouse = warehouse; + this.queryWatcher = queryWatcher; this.smileMapper = smileMapper; this.httpClient = httpClient; this.baseView = baseView; @@ -154,7 +156,7 @@ public class BrokerServerView implements TimelineServerView private DirectDruidClient makeDirectClient(DruidServer server) { - return new DirectDruidClient(warehouse, smileMapper, httpClient, server.getHost()); + return new DirectDruidClient(warehouse, queryWatcher, smileMapper, httpClient, server.getHost()); } private QueryableDruidServer removeServer(DruidServer server) diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index c08cd9e2bd4..34584ee2395 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.dataformat.smile.SmileFactory; +import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.google.common.io.Closeables; @@ -43,12 +44,15 @@ import com.metamx.http.client.HttpClient; import com.metamx.http.client.io.AppendableByteArrayInputStream; import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; import io.druid.query.BySegmentResultValueClass; import io.druid.query.Query; import io.druid.query.QueryInterruptedException; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.aggregation.MetricManipulatorFns; import org.jboss.netty.handler.codec.http.HttpChunk; @@ -61,6 +65,7 @@ import java.io.InputStream; import java.net.URL; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; @@ -74,6 +79,7 @@ public class DirectDruidClient implements QueryRunner private static final Map, Pair> typesMap = Maps.newConcurrentMap(); private final QueryToolChestWarehouse warehouse; + private final QueryWatcher queryWatcher; private final ObjectMapper objectMapper; private final HttpClient httpClient; private final String host; @@ -83,12 +89,14 @@ public class DirectDruidClient implements QueryRunner public DirectDruidClient( QueryToolChestWarehouse warehouse, + QueryWatcher queryWatcher, ObjectMapper objectMapper, HttpClient httpClient, String host ) { this.warehouse = warehouse; + this.queryWatcher = queryWatcher; this.objectMapper = objectMapper; this.httpClient = httpClient; this.host = host; @@ -103,7 +111,7 @@ public class DirectDruidClient implements QueryRunner } @Override - public Sequence run(Query query) + public Sequence run(final Query query) { QueryToolChest> toolChest = warehouse.getToolChest(query); boolean isBySegment = query.getContextBySegment(false); @@ -128,6 +136,7 @@ public class DirectDruidClient implements QueryRunner final ListenableFuture future; final String url = String.format("http://%s/druid/v2/", host); + final String cancelUrl = String.format("http://%s/druid/v2/%s", host, query.getId()); try { log.debug("Querying url[%s]", url); @@ -175,6 +184,9 @@ public class DirectDruidClient implements QueryRunner } } ); + + queryWatcher.registerQuery(query, future); + openConnections.getAndIncrement(); Futures.addCallback( future, new FutureCallback() @@ -189,6 +201,27 @@ public class DirectDruidClient implements QueryRunner public void onFailure(Throwable t) { openConnections.getAndDecrement(); + if (future.isCancelled()) { + // forward the cancellation to underlying queriable node + try { + StatusResponseHolder res = httpClient + .delete(new URL(cancelUrl)) + .setContent(objectMapper.writeValueAsBytes(query)) + .setHeader(HttpHeaders.Names.CONTENT_TYPE, isSmile ? "application/smile" : "application/json") + .go(new StatusResponseHandler(Charsets.UTF_8)) + .get(); + if (res.getStatus().getCode() >= 500) { + throw new RE( + "Error cancelling query[%s]: queriable node returned status[%d] [%s].", + res.getStatus().getCode(), + res.getStatus().getReasonPhrase() + ); + } + } + catch (IOException | ExecutionException | InterruptedException e) { + Throwables.propagate(e); + } + } } } ); @@ -197,7 +230,7 @@ public class DirectDruidClient implements QueryRunner throw Throwables.propagate(e); } - Sequence retVal = new BaseSequence>( + Sequence retVal = new BaseSequence<>( new BaseSequence.IteratorMaker>() { @Override @@ -296,14 +329,11 @@ public class DirectDruidClient implements QueryRunner objectCodec = jp.getCodec(); } } - catch (IOException e) { + catch (IOException | InterruptedException | ExecutionException e) { throw new RE(e, "Failure getting results from[%s]", url); } - catch (InterruptedException e) { - throw new RE(e, "Failure getting results from[%s]", url); - } - catch (ExecutionException e) { - throw new RE(e, "Failure getting results from[%s]", url); + catch (CancellationException e) { + throw new QueryInterruptedException(); } } } diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index 1e6ea06607f..c97657b4e35 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -73,7 +73,6 @@ import java.util.UUID; public class QueryResource { private static final EmittingLogger log = new EmittingLogger(QueryResource.class); - private static final Charset UTF8 = Charset.forName("UTF-8"); private static final Joiner COMMA_JOIN = Joiner.on(","); public static final String APPLICATION_SMILE = "application/smile"; public static final String APPLICATION_JSON = "application/json"; diff --git a/server/src/test/java/io/druid/client/DirectDruidClientTest.java b/server/src/test/java/io/druid/client/DirectDruidClientTest.java index aba91657686..84a80058d35 100644 --- a/server/src/test/java/io/druid/client/DirectDruidClientTest.java +++ b/server/src/test/java/io/druid/client/DirectDruidClientTest.java @@ -21,18 +21,24 @@ package io.druid.client; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.http.client.HttpClient; import com.metamx.http.client.Request; import com.metamx.http.client.RequestBuilder; +import com.metamx.http.client.response.StatusResponseHolder; import io.druid.client.selector.ConnectionCountServerSelectorStrategy; import io.druid.client.selector.HighestPriorityTierSelectorStrategy; import io.druid.client.selector.QueryableDruidServer; import io.druid.client.selector.ServerSelector; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; +import io.druid.query.Query; +import io.druid.query.QueryInterruptedException; +import io.druid.query.QueryWatcher; import io.druid.query.ReflectionQueryToolChestWarehouse; import io.druid.query.Result; import io.druid.query.timeboundary.TimeBoundaryQuery; @@ -41,11 +47,13 @@ import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.timeout.ReadTimeoutException; import org.joda.time.DateTime; import org.joda.time.Interval; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -54,17 +62,22 @@ import java.util.List; public class DirectDruidClientTest { - private HttpClient httpClient; - - @Before - public void setUp() throws Exception + public static final QueryWatcher DUMMY_WATCHER = new QueryWatcher() { - httpClient = EasyMock.createMock(HttpClient.class); - } + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }; + + @Rule + public ExpectedException thrown = ExpectedException.none(); @Test public void testRun() throws Exception { + HttpClient httpClient = EasyMock.createMock(HttpClient.class); RequestBuilder requestBuilder = new RequestBuilder(httpClient, HttpMethod.POST, new URL("http://foo.com")); EasyMock.expect(httpClient.post(EasyMock.anyObject())).andReturn(requestBuilder).atLeastOnce(); @@ -93,12 +106,14 @@ public class DirectDruidClientTest DirectDruidClient client1 = new DirectDruidClient( new ReflectionQueryToolChestWarehouse(), + DUMMY_WATCHER, new DefaultObjectMapper(), httpClient, "foo" ); DirectDruidClient client2 = new DirectDruidClient( new ReflectionQueryToolChestWarehouse(), + DUMMY_WATCHER, new DefaultObjectMapper(), httpClient, "foo2" @@ -149,4 +164,65 @@ public class DirectDruidClientTest EasyMock.verify(httpClient); } + + @Test + public void testCancel() throws Exception + { + HttpClient httpClient = EasyMock.createStrictMock(HttpClient.class); + EasyMock.expect(httpClient.post(EasyMock.anyObject())).andReturn( + new RequestBuilder(httpClient, HttpMethod.POST, new URL("http://foo.com")) + ).once(); + + ListenableFuture cancelledFuture = Futures.immediateCancelledFuture(); + EasyMock.expect(httpClient.go(EasyMock.anyObject())).andReturn(cancelledFuture).once(); + + EasyMock.expect(httpClient.delete(EasyMock.anyObject())) + .andReturn(new RequestBuilder(httpClient, HttpMethod.DELETE, new URL("http://foo.com/delete"))) + .once(); + SettableFuture cancellationFuture = SettableFuture.create(); + EasyMock.expect(httpClient.go(EasyMock.anyObject())).andReturn(cancellationFuture).once(); + + EasyMock.replay(httpClient); + + final ServerSelector serverSelector = new ServerSelector( + new DataSegment( + "test", + new Interval("2013-01-01/2013-01-02"), + new DateTime("2013-01-01").toString(), + Maps.newHashMap(), + Lists.newArrayList(), + Lists.newArrayList(), + new NoneShardSpec(), + 0, + 0L + ), + new HighestPriorityTierSelectorStrategy(new ConnectionCountServerSelectorStrategy()) + ); + + DirectDruidClient client1 = new DirectDruidClient( + new ReflectionQueryToolChestWarehouse(), + DUMMY_WATCHER, + new DefaultObjectMapper(), + httpClient, + "foo" + ); + + QueryableDruidServer queryableDruidServer1 = new QueryableDruidServer( + new DruidServer("test1", "localhost", 0, "historical", DruidServer.DEFAULT_TIER, 0), + client1 + ); + serverSelector.addServer(queryableDruidServer1); + + TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource("test").build(); + + cancellationFuture.set(new StatusResponseHolder(HttpResponseStatus.OK, new StringBuilder("cancelled"))); + Sequence results = client1.run(query); + Assert.assertEquals(0, client1.getNumOpenConnections()); + + + thrown.expect(QueryInterruptedException.class); + Assert.assertTrue(Sequences.toList(results, Lists.newArrayList()).isEmpty()); + + EasyMock.verify(httpClient); + } } From 855c66c9ad13e0b133597e0324ffaf761c99bbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 2 Jun 2014 18:29:42 -0700 Subject: [PATCH 28/39] less stack traces when cancelling queries --- .../query/ChainedExecutionQueryRunner.java | 5 ++-- .../io/druid/client/DirectDruidClient.java | 2 +- .../java/io/druid/server/QueryResource.java | 26 +++++++++++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index c7ed29f1ddf..256cefa33a0 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -136,6 +136,9 @@ public class ChainedExecutionQueryRunner implements QueryRunner return retVal; } + catch (QueryInterruptedException e) { + throw Throwables.propagate(e); + } catch (Exception e) { log.error(e, "Exception with one of the sequences!"); throw Throwables.propagate(e); @@ -166,11 +169,9 @@ public class ChainedExecutionQueryRunner implements QueryRunner throw new QueryInterruptedException("Query interrupted"); } catch(CancellationException e) { - log.warn(e, "Query cancelled, query id [%s]", query.getId()); throw new QueryInterruptedException("Query cancelled"); } catch(TimeoutException e) { - log.warn(e, "Query timeout, query id [%s]", query.getId()); throw new QueryInterruptedException("Query timeout"); } catch (ExecutionException e) { diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index 34584ee2395..b6030f9755b 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -333,7 +333,7 @@ public class DirectDruidClient implements QueryRunner throw new RE(e, "Failure getting results from[%s]", url); } catch (CancellationException e) { - throw new QueryInterruptedException(); + throw new QueryInterruptedException("Query cancelled"); } } } diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index c97657b4e35..33bdd519c83 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -125,7 +125,7 @@ public class QueryResource final long start = System.currentTimeMillis(); Query query = null; byte[] requestQuery = null; - String queryId; + String queryId = null; final boolean isSmile = APPLICATION_SMILE.equals(req.getContentType()); @@ -212,6 +212,28 @@ public class QueryResource .build(); } } + catch (QueryInterruptedException e) { + try { + log.info("%s [%s]", e.getMessage(), queryId); + requestLogger.log( + new RequestLogLine( + new DateTime(), + req.getRemoteAddr(), + query, + new QueryStats(ImmutableMap.of("success", false, "interrupted", true, "reason", e.toString())) + ) + ); + } catch (Exception e2) { + log.error(e2, "Unable to log query [%s]!", query); + } + return Response.serverError().entity( + jsonWriter.writeValueAsString( + ImmutableMap.of( + "error", e.getMessage() + ) + ) + ).build(); + } catch (Exception e) { final String queryString = query == null @@ -243,7 +265,7 @@ public class QueryResource return Response.serverError().entity( jsonWriter.writeValueAsString( ImmutableMap.of( - "error", (e.getMessage() == null) ? "null Exception" : e.getMessage() + "error", e.getMessage() == null ? "null exception" : e.getMessage() ) ) ).build(); From d0f9c438f826f6ebe072bb193a80e8b5757d0c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 2 Jun 2014 17:39:08 -0700 Subject: [PATCH 29/39] proper query cancellation tests --- .../ChainedExecutionQueryRunnerTest.java | 115 +++++++++++------- .../io/druid/query/QueryRunnerTestHelper.java | 11 ++ .../java/io/druid/query/TestQueryRunners.java | 13 +- .../druid/query/topn/TopNQueryRunnerTest.java | 18 +-- .../druid/query/topn/TopNUnionQueryTest.java | 9 +- .../druid/client/DirectDruidClientTest.java | 16 +-- 6 files changed, 107 insertions(+), 75 deletions(-) diff --git a/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java index 445bea9cf53..b5391605d32 100644 --- a/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java @@ -20,7 +20,6 @@ package io.druid.query; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.concurrent.ExecutorServiceConfig; @@ -29,18 +28,22 @@ import com.metamx.common.guava.Sequences; import com.metamx.common.lifecycle.Lifecycle; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; -import org.junit.Ignore; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.junit.Assert; import org.junit.Test; -import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class ChainedExecutionQueryRunnerTest { - @Test @Ignore + @Test public void testQueryCancellation() throws Exception { ExecutorService exec = PrioritizedExecutorService.create( @@ -63,25 +66,36 @@ public class ChainedExecutionQueryRunnerTest final CountDownLatch queriesStarted = new CountDownLatch(2); final CountDownLatch queryIsRegistered = new CountDownLatch(1); - final Map queries = Maps.newHashMap(); - QueryWatcher watcher = new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - queries.put(query, future); - queryIsRegistered.countDown(); - } - }; + Capture capturedFuture = new Capture<>(); + QueryWatcher watcher = EasyMock.createStrictMock(QueryWatcher.class); + watcher.registerQuery(EasyMock.anyObject(), EasyMock.and(EasyMock.anyObject(), EasyMock.capture(capturedFuture))); + EasyMock.expectLastCall() + .andAnswer( + new IAnswer() + { + @Override + public Void answer() throws Throwable + { + queryIsRegistered.countDown(); + return null; + } + } + ) + .once(); + EasyMock.replay(watcher); + + DyingQueryRunner runner1 = new DyingQueryRunner(queriesStarted); + DyingQueryRunner runner2 = new DyingQueryRunner(queriesStarted); + DyingQueryRunner runner3 = new DyingQueryRunner(queriesStarted); ChainedExecutionQueryRunner chainedRunner = new ChainedExecutionQueryRunner<>( exec, Ordering.natural(), watcher, Lists.>newArrayList( - new DyingQueryRunner(1, queriesStarted), - new DyingQueryRunner(2, queriesStarted), - new DyingQueryRunner(3, queriesStarted) + runner1, + runner2, + runner3 ) ); @@ -93,7 +107,7 @@ public class ChainedExecutionQueryRunnerTest .build() ); - Future f = Executors.newFixedThreadPool(1).submit( + Future resultFuture = Executors.newFixedThreadPool(1).submit( new Runnable() { @Override @@ -104,45 +118,64 @@ public class ChainedExecutionQueryRunnerTest } ); - // wait for query to register - queryIsRegistered.await(); - queriesStarted.await(); + // wait for query to register and start + Assert.assertTrue(queryIsRegistered.await(1, TimeUnit.SECONDS)); + Assert.assertTrue(queriesStarted.await(1, TimeUnit.SECONDS)); // cancel the query - queries.values().iterator().next().cancel(true); - f.get(); + Assert.assertTrue(capturedFuture.hasCaptured()); + ListenableFuture future = capturedFuture.getValue(); + future.cancel(true); + + QueryInterruptedException cause = null; + try { + resultFuture.get(); + } catch(ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof QueryInterruptedException); + cause = (QueryInterruptedException)e.getCause(); + } + Assert.assertNotNull(cause); + Assert.assertTrue(future.isCancelled()); + Assert.assertTrue(runner1.hasStarted); + Assert.assertTrue(runner2.hasStarted); + Assert.assertFalse(runner3.hasStarted); + Assert.assertFalse(runner1.hasCompleted); + Assert.assertFalse(runner2.hasCompleted); + Assert.assertFalse(runner3.hasCompleted); + + EasyMock.verify(watcher); } private static class DyingQueryRunner implements QueryRunner { - private final int id; private final CountDownLatch latch; + private boolean hasStarted = false; + private boolean hasCompleted = false; - public DyingQueryRunner(int id, CountDownLatch latch) { - this.id = id; + public DyingQueryRunner(CountDownLatch latch) + { this.latch = latch; } @Override public Sequence run(Query query) { + hasStarted = true; latch.countDown(); - - int i = 0; - while (i >= 0) { - if(Thread.interrupted()) { - throw new QueryInterruptedException("I got killed"); - } - - // do a lot of work - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new QueryInterruptedException("I got killed"); - } - ++i; + if (Thread.interrupted()) { + throw new QueryInterruptedException("I got killed"); } - return Sequences.simple(Lists.newArrayList(i)); + + // do a lot of work + try { + Thread.sleep(500); + } + catch (InterruptedException e) { + throw new QueryInterruptedException("I got killed"); + } + + hasCompleted = true; + return Sequences.simple(Lists.newArrayList(123)); } } } diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index ffa6b02c236..954cb4fd5ee 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -21,6 +21,7 @@ package io.druid.query; import com.google.common.base.Function; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; @@ -53,6 +54,16 @@ import java.util.List; */ public class QueryRunnerTestHelper { + + public static final QueryWatcher DUMMY_QUERYWATCHER = new QueryWatcher() + { + @Override + public void registerQuery(Query query, ListenableFuture future) + { + + } + }; + public static final String segmentId = "testSegment"; public static final String dataSource = "testing"; public static final UnionDataSource unionDataSource = new UnionDataSource( diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index a858b5e0cdf..90d394f3e3b 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -41,14 +41,11 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new TopNQueryRunnerFactory(pool, new TopNQueryQueryToolChest(topNConfig), new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }); + QueryRunnerFactory factory = new TopNQueryRunnerFactory( + pool, + new TopNQueryQueryToolChest(topNConfig), + QueryRunnerTestHelper.DUMMY_QUERYWATCHER + ); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 39af4459794..f06258a9a09 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -72,14 +72,7 @@ public class TopNQueryRunnerTest new TopNQueryRunnerFactory( TestQueryRunners.getPool(), new TopNQueryQueryToolChest(new TopNQueryConfig()), - new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - } + QueryRunnerTestHelper.DUMMY_QUERYWATCHER ) ) ); @@ -97,14 +90,7 @@ public class TopNQueryRunnerTest } ), new TopNQueryQueryToolChest(new TopNQueryConfig()), - new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - } + QueryRunnerTestHelper.DUMMY_QUERYWATCHER ) ) ); diff --git a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java index 1fdc3b11cf5..d2e3e5dea73 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java @@ -23,9 +23,12 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; import io.druid.collections.StupidPool; +import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.TestQueryRunners; import io.druid.query.aggregation.AggregatorFactory; @@ -65,7 +68,8 @@ public class TopNUnionQueryTest QueryRunnerTestHelper.makeUnionQueryRunners( new TopNQueryRunnerFactory( TestQueryRunners.getPool(), - new TopNQueryQueryToolChest(new TopNQueryConfig()) + new TopNQueryQueryToolChest(new TopNQueryConfig()), + QueryRunnerTestHelper.DUMMY_QUERYWATCHER ) ) ); @@ -82,7 +86,8 @@ public class TopNUnionQueryTest } } ), - new TopNQueryQueryToolChest(new TopNQueryConfig()) + new TopNQueryQueryToolChest(new TopNQueryConfig()), + QueryRunnerTestHelper.DUMMY_QUERYWATCHER ) ) ); diff --git a/server/src/test/java/io/druid/client/DirectDruidClientTest.java b/server/src/test/java/io/druid/client/DirectDruidClientTest.java index 84a80058d35..b2396688900 100644 --- a/server/src/test/java/io/druid/client/DirectDruidClientTest.java +++ b/server/src/test/java/io/druid/client/DirectDruidClientTest.java @@ -44,16 +44,14 @@ import io.druid.query.Result; import io.druid.query.timeboundary.TimeBoundaryQuery; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; -import junit.framework.Assert; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.timeout.ReadTimeoutException; import org.joda.time.DateTime; import org.joda.time.Interval; -import org.junit.Rule; +import org.junit.Assert; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -71,9 +69,6 @@ public class DirectDruidClientTest } }; - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void testRun() throws Exception { @@ -220,8 +215,13 @@ public class DirectDruidClientTest Assert.assertEquals(0, client1.getNumOpenConnections()); - thrown.expect(QueryInterruptedException.class); - Assert.assertTrue(Sequences.toList(results, Lists.newArrayList()).isEmpty()); + QueryInterruptedException exception = null; + try { + Sequences.toList(results, Lists.newArrayList()); + } catch(QueryInterruptedException e) { + exception = e; + } + Assert.assertNotNull(exception); EasyMock.verify(httpClient); } From 97d5455f3a455fecbb1296f60fdf038ef040f862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 3 Jun 2014 11:15:03 -0700 Subject: [PATCH 30/39] properly kill timed out queries --- .../query/ChainedExecutionQueryRunner.java | 2 + .../ChainedExecutionQueryRunnerTest.java | 105 ++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index 256cefa33a0..8a5ed51a4df 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -172,6 +172,8 @@ public class ChainedExecutionQueryRunner implements QueryRunner throw new QueryInterruptedException("Query cancelled"); } catch(TimeoutException e) { + log.info("Query timeout, cancelling pending results for query id [%s]", query.getId()); + futures.cancel(true); throw new QueryInterruptedException("Query timeout"); } catch (ExecutionException e) { diff --git a/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java index b5391605d32..f2555dd7214 100644 --- a/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/ChainedExecutionQueryRunnerTest.java @@ -19,6 +19,7 @@ package io.druid.query; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.util.concurrent.ListenableFuture; @@ -146,6 +147,110 @@ public class ChainedExecutionQueryRunnerTest EasyMock.verify(watcher); } + @Test + public void testQueryTimeout() throws Exception + { + ExecutorService exec = PrioritizedExecutorService.create( + new Lifecycle(), new ExecutorServiceConfig() + { + @Override + public String getFormatString() + { + return "test"; + } + + @Override + public int getNumThreads() + { + return 2; + } + } + ); + + final CountDownLatch queriesStarted = new CountDownLatch(2); + final CountDownLatch queryIsRegistered = new CountDownLatch(1); + + Capture capturedFuture = new Capture<>(); + QueryWatcher watcher = EasyMock.createStrictMock(QueryWatcher.class); + watcher.registerQuery(EasyMock.anyObject(), EasyMock.and(EasyMock.anyObject(), EasyMock.capture(capturedFuture))); + EasyMock.expectLastCall() + .andAnswer( + new IAnswer() + { + @Override + public Void answer() throws Throwable + { + queryIsRegistered.countDown(); + return null; + } + } + ) + .once(); + + EasyMock.replay(watcher); + + DyingQueryRunner runner1 = new DyingQueryRunner(queriesStarted); + DyingQueryRunner runner2 = new DyingQueryRunner(queriesStarted); + DyingQueryRunner runner3 = new DyingQueryRunner(queriesStarted); + ChainedExecutionQueryRunner chainedRunner = new ChainedExecutionQueryRunner<>( + exec, + Ordering.natural(), + watcher, + Lists.>newArrayList( + runner1, + runner2, + runner3 + ) + ); + + final Sequence seq = chainedRunner.run( + Druids.newTimeseriesQueryBuilder() + .dataSource("test") + .intervals("2014/2015") + .aggregators(Lists.newArrayList(new CountAggregatorFactory("count"))) + .context(ImmutableMap.of("timeout", (100), "queryId", "test")) + .build() + ); + + Future resultFuture = Executors.newFixedThreadPool(1).submit( + new Runnable() + { + @Override + public void run() + { + Sequences.toList(seq, Lists.newArrayList()); + } + } + ); + + // wait for query to register and start + Assert.assertTrue(queryIsRegistered.await(1, TimeUnit.SECONDS)); + Assert.assertTrue(queriesStarted.await(1, TimeUnit.SECONDS)); + + // cancel the query + Assert.assertTrue(capturedFuture.hasCaptured()); + ListenableFuture future = capturedFuture.getValue(); + + QueryInterruptedException cause = null; + try { + resultFuture.get(); + } catch(ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof QueryInterruptedException); + Assert.assertEquals("Query timeout", e.getCause().getMessage()); + cause = (QueryInterruptedException)e.getCause(); + } + Assert.assertNotNull(cause); + Assert.assertTrue(future.isCancelled()); + Assert.assertTrue(runner1.hasStarted); + Assert.assertTrue(runner2.hasStarted); + Assert.assertFalse(runner3.hasStarted); + Assert.assertFalse(runner1.hasCompleted); + Assert.assertFalse(runner2.hasCompleted); + Assert.assertFalse(runner3.hasCompleted); + + EasyMock.verify(watcher); + } + private static class DyingQueryRunner implements QueryRunner { private final CountDownLatch latch; From 99c9a2cf05fc7065a333a6b943cfb3e73fcd29e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 3 Jun 2014 11:35:26 -0700 Subject: [PATCH 31/39] make sure to close yielder in MetricsEmittingQueryRunner --- .../query/MetricsEmittingQueryRunner.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java index 8425aa97fe2..dbad443cb36 100644 --- a/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java @@ -167,18 +167,20 @@ public class MetricsEmittingQueryRunner implements QueryRunner @Override public void close() throws IOException { - if (!isDone() && builder.getUser10() == null) { - builder.setUser10("short"); + try { + if (!isDone() && builder.getUser10() == null) { + builder.setUser10("short"); + } + + long timeTaken = System.currentTimeMillis() - startTime; + emitter.emit(builder.build("query/time", timeTaken)); + + if (creationTime > 0) { + emitter.emit(builder.build("query/wait", startTime - creationTime)); + } + } finally { + yielder.close(); } - - long timeTaken = System.currentTimeMillis() - startTime; - emitter.emit(builder.build("query/time", timeTaken)); - - if(creationTime > 0) { - emitter.emit(builder.build("query/wait", startTime - creationTime)); - } - - yielder.close(); } }; } From c08002aa4d9b2c5892ee090833d5cd33ac827e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 3 Jun 2014 17:32:14 -0700 Subject: [PATCH 32/39] interrupt queries on incremental indexer --- .../segment/incremental/IncrementalIndexStorageAdapter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 3fe807b2761..057754e6087 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -201,6 +201,10 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter } while (baseIter.hasNext()) { + if (Thread.interrupted()) { + throw new QueryInterruptedException(); + } + currEntry.set(baseIter.next()); if (filterMatcher.matches()) { From b84884ab7677a34485a7bc6d961cf8d1d29eb043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 9 Jun 2014 13:48:01 -0700 Subject: [PATCH 33/39] remove methods used only for testing --- .../select/SelectQueryRunnerFactory.java | 15 ---------- .../TimeseriesQueryRunnerFactory.java | 19 ------------ .../io/druid/query/QueryRunnerTestHelper.java | 2 +- .../java/io/druid/query/TestQueryRunners.java | 30 +++++++------------ .../query/search/SearchQueryRunnerTest.java | 12 +++----- .../query/select/SelectQueryRunnerTest.java | 10 ++++++- .../TimeBoundaryQueryRunnerTest.java | 9 +----- .../TimeSeriesUnionQueryRunnerTest.java | 7 ++++- .../TimeseriesQueryRunnerBonusTest.java | 9 +++++- .../timeseries/TimeseriesQueryRunnerTest.java | 7 ++++- .../druid/query/topn/TopNQueryRunnerTest.java | 7 ++--- .../druid/query/topn/TopNUnionQueryTest.java | 7 ++--- .../filter/SpatialFilterBonusTest.java | 20 +++++++++++-- .../segment/filter/SpatialFilterTest.java | 20 +++++++++++-- .../server/coordination/ServerManager.java | 1 - .../druid/client/DirectDruidClientTest.java | 16 +++------- 16 files changed, 88 insertions(+), 103 deletions(-) diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java index a1fa77dabd5..72cce700a6d 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryRunnerFactory.java @@ -41,21 +41,6 @@ import java.util.concurrent.ExecutorService; public class SelectQueryRunnerFactory implements QueryRunnerFactory, SelectQuery> { - public static SelectQueryRunnerFactory create(ObjectMapper jsonMapper) - { - return new SelectQueryRunnerFactory( - new SelectQueryQueryToolChest(new QueryConfig(), jsonMapper), - new SelectQueryEngine(), - new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - } - } - ); - } - private final SelectQueryQueryToolChest toolChest; private final SelectQueryEngine engine; private final QueryWatcher queryWatcher; diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java index 726bc20bb43..724d4818226 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java @@ -41,25 +41,6 @@ import java.util.concurrent.ExecutorService; public class TimeseriesQueryRunnerFactory implements QueryRunnerFactory, TimeseriesQuery> { - /** - * Use only for testing - * @return - */ - public static TimeseriesQueryRunnerFactory create() - { - return new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(new QueryConfig()), - new TimeseriesQueryEngine(), - new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - } - } - ); - } - private final TimeseriesQueryQueryToolChest toolChest; private final TimeseriesQueryEngine engine; private final QueryWatcher queryWatcher; diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 954cb4fd5ee..55c29752ac1 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -55,7 +55,7 @@ import java.util.List; public class QueryRunnerTestHelper { - public static final QueryWatcher DUMMY_QUERYWATCHER = new QueryWatcher() + public static final QueryWatcher NOOP_QUERYWATCHER = new QueryWatcher() { @Override public void registerQuery(Query query, ListenableFuture future) diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index 90d394f3e3b..f50e81d038e 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -7,6 +7,8 @@ import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; import io.druid.query.search.search.SearchQueryConfig; import io.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesQueryEngine; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.topn.TopNQueryConfig; import io.druid.query.topn.TopNQueryQueryToolChest; @@ -44,7 +46,7 @@ public class TestQueryRunners QueryRunnerFactory factory = new TopNQueryRunnerFactory( pool, new TopNQueryQueryToolChest(topNConfig), - QueryRunnerTestHelper.DUMMY_QUERYWATCHER + QueryRunnerTestHelper.NOOP_QUERYWATCHER ); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), @@ -56,7 +58,12 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + QueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() @@ -67,14 +74,7 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig()), new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }); + QueryRunnerFactory factory = new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig()), QueryRunnerTestHelper.NOOP_QUERYWATCHER); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() @@ -85,18 +85,10 @@ public class TestQueryRunners Segment adapter ) { - QueryRunnerFactory factory = new TimeBoundaryQueryRunnerFactory(new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }); + QueryRunnerFactory factory = new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() ); } - } diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 0740333eed5..c69ee1c5a27 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -59,14 +59,10 @@ public class SearchQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig()), new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }) + new SearchQueryRunnerFactory( + new SearchQueryQueryToolChest(new SearchQueryConfig()), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) ); } diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 5015239870e..07f99165873 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -22,11 +22,15 @@ package io.druid.query.select; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.ISE; import com.metamx.common.guava.Sequences; import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Query; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.TableDataSource; import io.druid.query.filter.SelectorDimFilter; @@ -54,7 +58,11 @@ public class SelectQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - SelectQueryRunnerFactory.create(new DefaultObjectMapper()) + new SelectQueryRunnerFactory( + new SelectQueryQueryToolChest(new QueryConfig(), new DefaultObjectMapper()), + new SelectQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) ); } diff --git a/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java index de5ac1281b2..7bc499dca80 100644 --- a/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java @@ -46,14 +46,7 @@ public class TimeBoundaryQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - new TimeBoundaryQueryRunnerFactory(new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }) + new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER) ); } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java index 36e1fc13955..17d61908c3c 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; import io.druid.query.Druids; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; @@ -46,7 +47,11 @@ public class TimeSeriesUnionQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeUnionQueryRunners( - TimeseriesQueryRunnerFactory.create() + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) ); } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java index d1497a19026..67c91b4be40 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java @@ -28,8 +28,10 @@ import io.druid.granularity.QueryGranularity; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; @@ -87,7 +89,12 @@ public class TimeseriesQueryRunnerBonusTest private static List> runTimeseriesCount(IncrementalIndex index) { - final QueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + final QueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + final QueryRunner> runner = makeQueryRunner( factory, new IncrementalIndexSegment(index, null) diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index cfc26c4326a..708a7de1054 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -26,6 +26,7 @@ import com.metamx.common.guava.Sequences; import io.druid.granularity.PeriodGranularity; import io.druid.granularity.QueryGranularity; import io.druid.query.Druids; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; @@ -62,7 +63,11 @@ public class TimeseriesQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - TimeseriesQueryRunnerFactory.create() + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) ); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index f06258a9a09..09d383168cf 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -23,15 +23,12 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ListenableFuture; import com.metamx.common.guava.Sequences; import io.druid.collections.StupidPool; import io.druid.query.BySegmentResultValueClass; import io.druid.query.Druids; -import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; -import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.TestQueryRunners; import io.druid.query.aggregation.AggregatorFactory; @@ -72,7 +69,7 @@ public class TopNQueryRunnerTest new TopNQueryRunnerFactory( TestQueryRunners.getPool(), new TopNQueryQueryToolChest(new TopNQueryConfig()), - QueryRunnerTestHelper.DUMMY_QUERYWATCHER + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) ) ); @@ -90,7 +87,7 @@ public class TopNQueryRunnerTest } ), new TopNQueryQueryToolChest(new TopNQueryConfig()), - QueryRunnerTestHelper.DUMMY_QUERYWATCHER + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) ) ); diff --git a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java index d2e3e5dea73..7dc7b645cad 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java @@ -23,12 +23,9 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ListenableFuture; import io.druid.collections.StupidPool; -import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; -import io.druid.query.QueryWatcher; import io.druid.query.Result; import io.druid.query.TestQueryRunners; import io.druid.query.aggregation.AggregatorFactory; @@ -69,7 +66,7 @@ public class TopNUnionQueryTest new TopNQueryRunnerFactory( TestQueryRunners.getPool(), new TopNQueryQueryToolChest(new TopNQueryConfig()), - QueryRunnerTestHelper.DUMMY_QUERYWATCHER + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) ) ); @@ -87,7 +84,7 @@ public class TopNUnionQueryTest } ), new TopNQueryQueryToolChest(new TopNQueryConfig()), - QueryRunnerTestHelper.DUMMY_QUERYWATCHER + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) ) ); diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java index 0eb327972ee..c8155526a89 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java @@ -29,13 +29,17 @@ import io.druid.data.input.impl.SpatialDimensionSchema; import io.druid.granularity.QueryGranularity; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.filter.SpatialDimFilter; import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryEngine; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; import io.druid.segment.IncrementalIndexSegment; @@ -434,7 +438,12 @@ public class SpatialFilterBonusTest ) ); try { - TimeseriesQueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + TimeseriesQueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + QueryRunner runner = new FinalizeResultsQueryRunner( factory.createRunner(segment), factory.getToolchest() @@ -516,7 +525,12 @@ public class SpatialFilterBonusTest ) ); try { - TimeseriesQueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + TimeseriesQueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + QueryRunner runner = new FinalizeResultsQueryRunner( factory.createRunner(segment), factory.getToolchest() @@ -528,4 +542,4 @@ public class SpatialFilterBonusTest throw Throwables.propagate(e); } } -} \ No newline at end of file +} diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index d342c12c577..84df58a260d 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -29,13 +29,17 @@ import io.druid.data.input.impl.SpatialDimensionSchema; import io.druid.granularity.QueryGranularity; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryConfig; import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.filter.SpatialDimFilter; import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryEngine; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; import io.druid.segment.IncrementalIndexSegment; @@ -464,7 +468,12 @@ public class SpatialFilterTest ) ); try { - TimeseriesQueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + TimeseriesQueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + QueryRunner runner = new FinalizeResultsQueryRunner( factory.createRunner(segment), factory.getToolchest() @@ -546,7 +555,12 @@ public class SpatialFilterTest ) ); try { - TimeseriesQueryRunnerFactory factory = TimeseriesQueryRunnerFactory.create(); + TimeseriesQueryRunnerFactory factory = new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(new QueryConfig()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + QueryRunner runner = new FinalizeResultsQueryRunner( factory.createRunner(segment), factory.getToolchest() @@ -558,4 +572,4 @@ public class SpatialFilterTest throw Throwables.propagate(e); } } -} \ No newline at end of file +} diff --git a/server/src/main/java/io/druid/server/coordination/ServerManager.java b/server/src/main/java/io/druid/server/coordination/ServerManager.java index 537cc0145fb..6bc703297e5 100644 --- a/server/src/main/java/io/druid/server/coordination/ServerManager.java +++ b/server/src/main/java/io/druid/server/coordination/ServerManager.java @@ -21,7 +21,6 @@ package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; -import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; diff --git a/server/src/test/java/io/druid/client/DirectDruidClientTest.java b/server/src/test/java/io/druid/client/DirectDruidClientTest.java index b2396688900..4ad8ca5cd51 100644 --- a/server/src/test/java/io/druid/client/DirectDruidClientTest.java +++ b/server/src/test/java/io/druid/client/DirectDruidClientTest.java @@ -38,6 +38,7 @@ import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; import io.druid.query.Query; import io.druid.query.QueryInterruptedException; +import io.druid.query.QueryRunnerTestHelper; import io.druid.query.QueryWatcher; import io.druid.query.ReflectionQueryToolChestWarehouse; import io.druid.query.Result; @@ -60,15 +61,6 @@ import java.util.List; public class DirectDruidClientTest { - public static final QueryWatcher DUMMY_WATCHER = new QueryWatcher() - { - @Override - public void registerQuery(Query query, ListenableFuture future) - { - - } - }; - @Test public void testRun() throws Exception { @@ -101,14 +93,14 @@ public class DirectDruidClientTest DirectDruidClient client1 = new DirectDruidClient( new ReflectionQueryToolChestWarehouse(), - DUMMY_WATCHER, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, new DefaultObjectMapper(), httpClient, "foo" ); DirectDruidClient client2 = new DirectDruidClient( new ReflectionQueryToolChestWarehouse(), - DUMMY_WATCHER, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, new DefaultObjectMapper(), httpClient, "foo2" @@ -196,7 +188,7 @@ public class DirectDruidClientTest DirectDruidClient client1 = new DirectDruidClient( new ReflectionQueryToolChestWarehouse(), - DUMMY_WATCHER, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, new DefaultObjectMapper(), httpClient, "foo" From 8f7fd93491a6eb97661ebc7dbe7bf70683adeed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 9 Jun 2014 14:34:44 -0700 Subject: [PATCH 34/39] add comments --- .../java/io/druid/query/QueryWatcher.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/processing/src/main/java/io/druid/query/QueryWatcher.java b/processing/src/main/java/io/druid/query/QueryWatcher.java index 0a76a54f23a..36394b11e34 100644 --- a/processing/src/main/java/io/druid/query/QueryWatcher.java +++ b/processing/src/main/java/io/druid/query/QueryWatcher.java @@ -21,7 +21,27 @@ package io.druid.query; import com.google.common.util.concurrent.ListenableFuture; +/** + * This interface is in a very early stage and should not be considered stable. + * + * The purpose of the QueryWatcher is to give overall visibility into queries running + * or pending at the QueryRunner level. This is currently used to cancel all the + * parts of a pending query, but may be expanded in the future to offer more direct + * visibility into query execution and resource usage. + * + * QueryRunners executing any computation asynchronously must register their queries + * with the QueryWatcher. + * + */ public interface QueryWatcher { + /** + * QueryRunners must use this method to register any pending queries. + * + * The given future may have cancel(true) called at any time, if cancellation of this query has been requested. + * + * @param query a query, which may be a subset of a larger query, as long as the underlying queryId is unchanged + * @param future the future holding the execution status of the query + */ public void registerQuery(Query query, ListenableFuture future); } From 1fb9b21cf0c5206e1f8ccf6ea1d5f771157488e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 9 Jun 2014 17:22:50 -0700 Subject: [PATCH 35/39] async servlet delete support + cleanup --- .../io/druid/client/RoutingDruidClient.java | 22 +- .../server/AsyncQueryForwardingServlet.java | 529 +++++++++--------- 2 files changed, 280 insertions(+), 271 deletions(-) diff --git a/server/src/main/java/io/druid/client/RoutingDruidClient.java b/server/src/main/java/io/druid/client/RoutingDruidClient.java index 10170fcfb9e..79ae1c16f6d 100644 --- a/server/src/main/java/io/druid/client/RoutingDruidClient.java +++ b/server/src/main/java/io/druid/client/RoutingDruidClient.java @@ -22,6 +22,7 @@ package io.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMultimap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -30,7 +31,9 @@ import com.metamx.http.client.HttpClient; import com.metamx.http.client.response.HttpResponseHandler; import io.druid.guice.annotations.Client; import io.druid.query.Query; +import io.druid.server.QueryResource; import io.druid.server.router.Router; +import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.handler.codec.http.HttpHeaders; import javax.inject.Inject; @@ -68,7 +71,7 @@ public class RoutingDruidClient return openConnections.get(); } - public ListenableFuture post( + public ListenableFuture postQuery( String url, Query query, HttpResponseHandler responseHandler @@ -81,7 +84,7 @@ public class RoutingDruidClient future = httpClient .post(new URL(url)) .setContent(objectMapper.writeValueAsBytes(query)) - .setHeader(HttpHeaders.Names.CONTENT_TYPE, isSmile ? "application/smile" : "application/json") + .setHeader(HttpHeaders.Names.CONTENT_TYPE, isSmile ? QueryResource.APPLICATION_SMILE : QueryResource.APPLICATION_JSON) .go(responseHandler); openConnections.getAndIncrement(); @@ -125,4 +128,19 @@ public class RoutingDruidClient throw Throwables.propagate(e); } } + + public ListenableFuture delete( + String url, + HttpResponseHandler responseHandler + ) + { + try { + return httpClient + .delete(new URL(url)) + .go(responseHandler); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } } diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index 5360d529ad3..3d82cc013ea 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -21,7 +21,11 @@ package io.druid.server; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Function; import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; @@ -37,19 +41,20 @@ import io.druid.server.log.RequestLogger; import io.druid.server.router.QueryHostFinder; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpResponse; import org.joda.time.DateTime; +import javax.annotation.Nullable; import javax.servlet.AsyncContext; import javax.servlet.ServletException; -import javax.servlet.ServletOutputStream; import javax.servlet.annotation.WebServlet; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.Charset; +import java.util.Map; import java.util.UUID; /** @@ -59,8 +64,6 @@ import java.util.UUID; public class AsyncQueryForwardingServlet extends HttpServlet { private static final EmittingLogger log = new EmittingLogger(AsyncQueryForwardingServlet.class); - private static final Charset UTF8 = Charset.forName("UTF-8"); - private static final String DISPATCHED = "dispatched"; private static final Joiner COMMA_JOIN = Joiner.on(","); private final ObjectMapper jsonMapper; @@ -88,275 +91,161 @@ public class AsyncQueryForwardingServlet extends HttpServlet } @Override - protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) + protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { - OutputStream out = null; - AsyncContext ctx = null; - - try { - ctx = req.startAsync(req, resp); - final AsyncContext asyncContext = ctx; - - if (req.getAttribute(DISPATCHED) != null) { - return; - } - - out = resp.getOutputStream(); - final OutputStream outputStream = out; - - final String host = hostFinder.getDefaultHost(); - - final HttpResponseHandler responseHandler = new HttpResponseHandler() - { - @Override - public ClientResponse handleResponse(HttpResponse response) + final AsyncContext asyncContext = req.startAsync(req, res); + asyncContext.start( + new Runnable() { - resp.setStatus(response.getStatus().getCode()); - resp.setContentType("application/json"); - - try { - ChannelBuffer buf = response.getContent(); - buf.readBytes(outputStream, buf.readableBytes()); - } - catch (Exception e) { - asyncContext.complete(); - throw Throwables.propagate(e); - } - - return ClientResponse.finished(outputStream); - } - - @Override - public ClientResponse handleChunk( - ClientResponse clientResponse, HttpChunk chunk - ) - { - try { - ChannelBuffer buf = chunk.getContent(); - buf.readBytes(outputStream, buf.readableBytes()); - } - catch (Exception e) { - asyncContext.complete(); - throw Throwables.propagate(e); - } - return clientResponse; - } - - @Override - public ClientResponse done(ClientResponse clientResponse) - { - final OutputStream obj = clientResponse.getObj(); - try { - resp.flushBuffer(); - outputStream.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - finally { - asyncContext.complete(); - } - - return ClientResponse.finished(obj); - } - - @Override - public void exceptionCaught( - ClientResponse clientResponse, - Throwable e - ) - { - handleException(resp, asyncContext, e); - } - }; - - asyncContext.start( - new Runnable() + @Override + public void run() { - @Override - public void run() - { - routingDruidClient.get(makeUrl(host, req), responseHandler); + try { + final HttpResponseHandler responseHandler = new PassthroughHttpResponseHandler(asyncContext, jsonMapper); + + final String host = hostFinder.getDefaultHost(); + routingDruidClient.get(makeUrl(host, (HttpServletRequest) asyncContext.getRequest()), responseHandler); + } + catch (Exception e) { + handleException(jsonMapper, asyncContext, e); } } - ); - - asyncContext.dispatch(); - req.setAttribute(DISPATCHED, true); - } - catch (Exception e) { - handleException(resp, ctx, e); - } + } + ); } @Override - protected void doPost( - final HttpServletRequest req, final HttpServletResponse resp - ) throws ServletException, IOException + protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { - final long start = System.currentTimeMillis(); - Query query = null; - String queryId; - - final boolean isSmile = "application/smile".equals(req.getContentType()); - - final ObjectMapper objectMapper = isSmile ? smileMapper : jsonMapper; - - OutputStream out = null; - AsyncContext ctx = null; - - try { - ctx = req.startAsync(req, resp); - final AsyncContext asyncContext = ctx; - - if (req.getAttribute(DISPATCHED) != null) { - return; - } - - query = objectMapper.readValue(req.getInputStream(), Query.class); - queryId = query.getId(); - if (queryId == null) { - queryId = UUID.randomUUID().toString(); - query = query.withId(queryId); - } - - if (log.isDebugEnabled()) { - log.debug("Got query [%s]", query); - } - - out = resp.getOutputStream(); - final OutputStream outputStream = out; - - final String host = hostFinder.getHost(query); - - final Query theQuery = query; - final String theQueryId = queryId; - - final HttpResponseHandler responseHandler = new HttpResponseHandler() - { - @Override - public ClientResponse handleResponse(HttpResponse response) + final AsyncContext asyncContext = req.startAsync(req, res); + asyncContext.start( + new Runnable() { - resp.setStatus(response.getStatus().getCode()); - resp.setContentType("application/x-javascript"); - - try { - ChannelBuffer buf = response.getContent(); - buf.readBytes(outputStream, buf.readableBytes()); - } - catch (Exception e) { - asyncContext.complete(); - throw Throwables.propagate(e); - } - return ClientResponse.finished(outputStream); - } - - @Override - public ClientResponse handleChunk( - ClientResponse clientResponse, HttpChunk chunk - ) - { - try { - ChannelBuffer buf = chunk.getContent(); - buf.readBytes(outputStream, buf.readableBytes()); - } - catch (Exception e) { - asyncContext.complete(); - throw Throwables.propagate(e); - } - return clientResponse; - } - - @Override - public ClientResponse done(ClientResponse clientResponse) - { - final long requestTime = System.currentTimeMillis() - start; - - log.debug("Request time: %d", requestTime); - - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser2(DataSourceUtil.getMetricName(theQuery.getDataSource())) - .setUser4(theQuery.getType()) - .setUser5(COMMA_JOIN.join(theQuery.getIntervals())) - .setUser6(String.valueOf(theQuery.hasFilters())) - .setUser7(req.getRemoteAddr()) - .setUser8(theQueryId) - .setUser9(theQuery.getDuration().toPeriod().toStandardMinutes().toString()) - .build("request/time", requestTime) - ); - - final OutputStream obj = clientResponse.getObj(); - try { - requestLogger.log( - new RequestLogLine( - new DateTime(), - req.getRemoteAddr(), - theQuery, - new QueryStats(ImmutableMap.of("request/time", requestTime, "success", true)) - ) - ); - - resp.flushBuffer(); - outputStream.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - finally { - asyncContext.complete(); - } - - return ClientResponse.finished(obj); - } - - @Override - public void exceptionCaught( - ClientResponse clientResponse, - Throwable e - ) - { - handleException(resp, asyncContext, e); - } - }; - - asyncContext.start( - new Runnable() + @Override + public void run() { - @Override - public void run() - { - routingDruidClient.post(makeUrl(host, req), theQuery, responseHandler); + try { + final HttpResponseHandler responseHandler = new PassthroughHttpResponseHandler(asyncContext, jsonMapper); + + final String host = hostFinder.getDefaultHost(); + routingDruidClient.delete(makeUrl(host, (HttpServletRequest) asyncContext.getRequest()), responseHandler); + } + catch (Exception e) { + handleException(jsonMapper, asyncContext, e); } } - ); + } + ); + } - asyncContext.dispatch(); - req.setAttribute(DISPATCHED, true); - } - catch (Exception e) { - handleException(resp, ctx, e); + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException + { + final long start = System.currentTimeMillis(); + final AsyncContext asyncContext = req.startAsync(req, res); + asyncContext.start( + new Runnable() + { + @Override + public void run() + { + final HttpServletRequest request = (HttpServletRequest) asyncContext.getRequest(); - try { - requestLogger.log( - new RequestLogLine( - new DateTime(), - req.getRemoteAddr(), - query, - new QueryStats(ImmutableMap.of("success", false, "exception", e.toString())) - ) - ); - } - catch (Exception e2) { - log.error(e2, "Unable to log query [%s]!", query); - } + final boolean isSmile = QueryResource.APPLICATION_SMILE.equals(request.getContentType()); + final ObjectMapper objectMapper = isSmile ? smileMapper : jsonMapper; - log.makeAlert(e, "Exception handling request") - .addData("query", query) - .addData("peer", req.getRemoteAddr()) - .emit(); - } + Query inputQuery = null; + try { + inputQuery = objectMapper.readValue(request.getInputStream(), Query.class); + if (inputQuery.getId() == null) { + inputQuery = inputQuery.withId(UUID.randomUUID().toString()); + } + final Query query = inputQuery; + + if (log.isDebugEnabled()) { + log.debug("Got query [%s]", inputQuery); + } + + final HttpResponseHandler responseHandler = new PassthroughHttpResponseHandler( + asyncContext, + objectMapper + ) + { + @Override + public ClientResponse done(ClientResponse clientResponse) + { + final long requestTime = System.currentTimeMillis() - start; + log.debug("Request time: %d", requestTime); + + emitter.emit( + new ServiceMetricEvent.Builder() + .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) + .setUser4(query.getType()) + .setUser5(COMMA_JOIN.join(query.getIntervals())) + .setUser6(String.valueOf(query.hasFilters())) + .setUser7(request.getRemoteAddr()) + .setUser8(query.getId()) + .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()) + .build("request/time", requestTime) + ); + + try { + requestLogger.log( + new RequestLogLine( + new DateTime(), + request.getRemoteAddr(), + query, + new QueryStats( + ImmutableMap.of( + "request/time", + requestTime, + "success", + true + ) + ) + ) + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + return super.done(clientResponse); + } + }; + + routingDruidClient.postQuery( + makeUrl(hostFinder.getHost(inputQuery), request), + inputQuery, + responseHandler + ); + } + catch (Exception e) { + handleException(objectMapper, asyncContext, e); + + try { + requestLogger.log( + new RequestLogLine( + new DateTime(), + request.getRemoteAddr(), + inputQuery, + new QueryStats(ImmutableMap.of("success", false, "exception", e.toString())) + ) + ); + } + catch (Exception logError) { + log.error(logError, "Unable to log query [%s]!", inputQuery); + } + + log.makeAlert(e, "Exception handling request") + .addData("query", inputQuery) + .addData("peer", request.getRemoteAddr()) + .emit(); + } + } + } + ); } private String makeUrl(final String host, final HttpServletRequest req) @@ -370,24 +259,126 @@ public class AsyncQueryForwardingServlet extends HttpServlet return String.format("http://%s%s?%s", host, requestURI, queryString); } - private static void handleException(HttpServletResponse resp, AsyncContext ctx, Throwable e) + private static void handleException(ObjectMapper objectMapper, AsyncContext asyncContext, Throwable exception) { try { - final ServletOutputStream out = resp.getOutputStream(); - if (!resp.isCommitted()) { - resp.setStatus(500); - resp.resetBuffer(); - out.write((e.getMessage() == null) ? "Exception null".getBytes(UTF8) : e.getMessage().getBytes(UTF8)); - out.write("\n".getBytes(UTF8)); + HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse(); + if (!response.isCommitted()) { + final String errorMessage = exception.getMessage() == null ? "null exception" : exception.getMessage(); + + response.resetBuffer(); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + objectMapper.writeValue( + response.getOutputStream(), + ImmutableMap.of( + "error", errorMessage + ) + ); + } + response.flushBuffer(); + } + catch (IOException e) { + Throwables.propagate(e); + } + finally { + asyncContext.complete(); + } + } + + private static class PassthroughHttpResponseHandler implements HttpResponseHandler + { + private final AsyncContext asyncContext; + private final ObjectMapper objectMapper; + private final OutputStream outputStream; + + public PassthroughHttpResponseHandler(AsyncContext asyncContext, ObjectMapper objectMapper) throws IOException + { + this.asyncContext = asyncContext; + this.objectMapper = objectMapper; + this.outputStream = asyncContext.getResponse().getOutputStream(); + } + + protected void copyStatusHeaders(HttpResponse clientResponse) + { + final HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse(); + response.setStatus(clientResponse.getStatus().getCode()); + response.setContentType(clientResponse.headers().get(HttpHeaders.Names.CONTENT_TYPE)); + + FluentIterable.from(clientResponse.headers().entries()) + .filter(new Predicate>() + { + @Override + public boolean apply(@Nullable Map.Entry input) + { + return input.getKey().startsWith("X-Druid"); + } + } + ) + .transform( + new Function, Object>() + { + @Nullable + @Override + public Object apply(@Nullable Map.Entry input) + { + response.setHeader(input.getKey(), input.getValue()); + return null; + } + } + ) + .allMatch(Predicates.alwaysTrue()); + } + + @Override + public ClientResponse handleResponse(HttpResponse clientResponse) + { + copyStatusHeaders(clientResponse); + + try { + ChannelBuffer buf = clientResponse.getContent(); + buf.readBytes(outputStream, buf.readableBytes()); + } + catch (Exception e) { + throw Throwables.propagate(e); } - if (ctx != null) { - ctx.complete(); - } - resp.flushBuffer(); + return ClientResponse.finished(outputStream); } - catch (IOException e1) { - Throwables.propagate(e1); + + @Override + public ClientResponse handleChunk( + ClientResponse clientResponse, HttpChunk chunk + ) + { + try { + ChannelBuffer buf = chunk.getContent(); + buf.readBytes(outputStream, buf.readableBytes()); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + return clientResponse; + } + + @Override + public ClientResponse done(ClientResponse clientResponse) + { + asyncContext.complete(); + return ClientResponse.finished(clientResponse.getObj()); + } + + @Override + public void exceptionCaught( + ClientResponse clientResponse, + Throwable e + ) + { + // throwing an exception here may cause resource leak + try { + handleException(objectMapper, asyncContext, e); + } catch(Exception err) { + log.error(err, "Unable to handle exception response"); + } } } } From ae0e36f3ebf91aa66a45062c5a44ebd88d643b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 2 Jun 2014 15:42:55 -0700 Subject: [PATCH 36/39] warn glacier objects cannot be moved --- pom.xml | 9 ++++++++- .../io/druid/storage/s3/S3DataSegmentMover.java | 4 ++++ .../io/druid/storage/s3/S3StorageDruidModule.java | 5 ----- .../io/druid/storage/s3/S3DataSegmentMoverTest.java | 13 +++++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 2116d7f617f..1c9c5664e47 100644 --- a/pom.xml +++ b/pom.xml @@ -393,7 +393,14 @@ net.java.dev.jets3t jets3t - 0.9.0 + 0.9.1 + + + + com.centerkey.utils + barebonesbrowserlaunch + + org.apache.httpcomponents diff --git a/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index d259ab185d2..ea585a0be44 100644 --- a/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -120,6 +120,10 @@ public class S3DataSegmentMover implements DataSegmentMover if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) { log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path); + } else if (s3Client.getObjectDetails(s3Bucket, s3Path) + .getStorageClass() + .equals(S3Object.STORAGE_CLASS_GLACIER)) { + log.warn("Cannot move file[s3://%s/%s] of storage class glacier."); } else { log.info( "Moving file[s3://%s/%s] to [s3://%s/%s]", diff --git a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index d30f49f976a..21faf74db91 100644 --- a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -73,11 +73,6 @@ public class S3StorageDruidModule implements DruidModule @LazySingleton public RestS3Service getRestS3Service(AWSCredentials credentials) { - try { return new RestS3Service(credentials); - } - catch (S3ServiceException e) { - throw new ProvisionException("Unable to create a RestS3Service", e); - } } } diff --git a/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java b/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java index c13d22de5f1..9497d9a05fc 100644 --- a/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java +++ b/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java @@ -30,6 +30,7 @@ import io.druid.timeline.partition.NoneShardSpec; import org.jets3t.service.S3ServiceException; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; +import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; import org.jets3t.service.model.StorageObject; import org.joda.time.Interval; @@ -131,6 +132,18 @@ public class S3DataSegmentMoverTest return (objects != null && objects.contains(objectKey)); } + @Override + public StorageObject getObjectDetails(String bucketName, String objectKey) throws ServiceException + { + if (isObjectInBucket(bucketName, objectKey)) { + final S3Object object = new S3Object(objectKey); + object.setStorageClass(S3Object.STORAGE_CLASS_STANDARD); + return object; + } else { + return null; + } + } + @Override public Map moveObject( String sourceBucketName, From 6550cb1776101232cd17086052d46f00b2e3fd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 10 Jun 2014 10:15:31 -0700 Subject: [PATCH 37/39] groupBy query cancellation --- .../query/GroupByParallelQueryRunner.java | 108 +++++++++++------- .../groupby/GroupByQueryRunnerFactory.java | 41 +++++-- .../query/groupby/GroupByQueryRunnerTest.java | 1 + .../GroupByTimeseriesQueryRunnerTest.java | 1 + 4 files changed, 105 insertions(+), 46 deletions(-) diff --git a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java index 51c663c6a2e..c9b14b6314b 100644 --- a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java +++ b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java @@ -26,6 +26,10 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.Pair; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; @@ -39,37 +43,44 @@ import io.druid.segment.incremental.IncrementalIndex; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public class GroupByParallelQueryRunner implements QueryRunner { private static final Logger log = new Logger(GroupByParallelQueryRunner.class); private final Iterable> queryables; - private final ExecutorService exec; + private final ListeningExecutorService exec; private final Ordering ordering; private final Supplier configSupplier; + private final QueryWatcher queryWatcher; + public GroupByParallelQueryRunner( ExecutorService exec, Ordering ordering, Supplier configSupplier, + QueryWatcher queryWatcher, QueryRunner... queryables ) { - this(exec, ordering, configSupplier, Arrays.asList(queryables)); + this(exec, ordering, configSupplier, queryWatcher, Arrays.asList(queryables)); } public GroupByParallelQueryRunner( ExecutorService exec, Ordering ordering, Supplier configSupplier, + QueryWatcher queryWatcher, Iterable> queryables ) { - this.exec = exec; + this.exec = MoreExecutors.listeningDecorator(exec); this.ordering = ordering; + this.queryWatcher = queryWatcher; this.queryables = Iterables.unmodifiableIterable(Iterables.filter(queryables, Predicates.notNull())); this.configSupplier = configSupplier; } @@ -88,48 +99,67 @@ public class GroupByParallelQueryRunner implements QueryRunner if (Iterables.isEmpty(queryables)) { log.warn("No queryables found."); } - List> futures = Lists.newArrayList( - Iterables.transform( - queryables, - new Function, Future>() - { - @Override - public Future apply(final QueryRunner input) - { - return exec.submit( - new AbstractPrioritizedCallable(priority) - { - @Override - public Boolean call() throws Exception - { - try { - input.run(queryParam).accumulate(indexAccumulatorPair.lhs, indexAccumulatorPair.rhs); - return true; + ListenableFuture> futures = Futures.allAsList( + Lists.newArrayList( + Iterables.transform( + queryables, + new Function, ListenableFuture>() + { + @Override + public ListenableFuture apply(final QueryRunner input) + { + return exec.submit( + new AbstractPrioritizedCallable(priority) + { + @Override + public Boolean call() throws Exception + { + try { + input.run(queryParam).accumulate(indexAccumulatorPair.lhs, indexAccumulatorPair.rhs); + return true; + } + catch (QueryInterruptedException e) { + throw Throwables.propagate(e); + } + catch (Exception e) { + log.error(e, "Exception with one of the sequences!"); + throw Throwables.propagate(e); + } + } } - catch (Exception e) { - log.error(e, "Exception with one of the sequences!"); - throw Throwables.propagate(e); - } - } - } - ); - } - } + ); + } + } + ) ) ); // Let the runners complete - for (Future future : futures) { - try { - future.get(); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - catch (ExecutionException e) { - throw new RuntimeException(e); + try { + queryWatcher.registerQuery(query, futures); + final Number timeout = query.getContextValue("timeout", (Number) null); + if(timeout == null) { + futures.get(); + } else { + futures.get(timeout.longValue(), TimeUnit.MILLISECONDS); } } + catch (InterruptedException e) { + log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId()); + futures.cancel(true); + throw new QueryInterruptedException("Query interrupted"); + } + catch(CancellationException e) { + throw new QueryInterruptedException("Query cancelled"); + } + catch(TimeoutException e) { + log.info("Query timeout, cancelling pending results for query id [%s]", query.getId()); + futures.cancel(true); + throw new QueryInterruptedException("Query timeout"); + } + catch (ExecutionException e) { + throw Throwables.propagate(e.getCause()); + } return Sequences.simple(indexAccumulatorPair.lhs.iterableWithPostAggregations(null)); } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java index 714aad37925..e8634089c2f 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java @@ -24,42 +24,55 @@ import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.ExecutorExecutingSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import com.metamx.common.logger.Logger; import io.druid.data.input.Row; import io.druid.query.ConcatQueryRunner; import io.druid.query.GroupByParallelQueryRunner; import io.druid.query.Query; +import io.druid.query.QueryInterruptedException; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** */ public class GroupByQueryRunnerFactory implements QueryRunnerFactory { private final GroupByQueryEngine engine; + private final QueryWatcher queryWatcher; private final Supplier config; private final GroupByQueryQueryToolChest toolChest; + private static final Logger log = new Logger(GroupByQueryRunnerFactory.class); + @Inject public GroupByQueryRunnerFactory( GroupByQueryEngine engine, + QueryWatcher queryWatcher, Supplier config, GroupByQueryQueryToolChest toolChest ) { this.engine = engine; + this.queryWatcher = queryWatcher; this.config = config; this.toolChest = toolChest; } @@ -71,8 +84,10 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory mergeRunners(final ExecutorService queryExecutor, Iterable> queryRunners) + public QueryRunner mergeRunners(final ExecutorService exec, Iterable> queryRunners) { + // mergeRunners should take ListeningExecutorService at some point + final ListeningExecutorService queryExecutor = MoreExecutors.listeningDecorator(exec); if (config.get().isSingleThreaded()) { return new ConcatQueryRunner( Sequences.map( @@ -88,7 +103,7 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory run(final Query query) { - Future> future = queryExecutor.submit( + ListenableFuture> future = queryExecutor.submit( new Callable>() { @Override @@ -102,13 +117,25 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory Date: Tue, 10 Jun 2014 10:24:18 -0700 Subject: [PATCH 38/39] segmentMetadata query cancellation --- .../SegmentMetadataQueryRunnerFactory.java | 48 +++++++++++++++---- .../query/metadata/SegmentAnalyzerTest.java | 3 +- .../metadata/SegmentMetadataQueryTest.java | 2 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index 9fa393c5c2f..c8e7208638c 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -22,15 +22,21 @@ package io.druid.query.metadata; import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.Maps; -import com.metamx.common.guava.ExecutorExecutingSequence; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.inject.Inject; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import com.metamx.common.logger.Logger; import io.druid.query.AbstractPrioritizedCallable; import io.druid.query.ConcatQueryRunner; import io.druid.query.Query; +import io.druid.query.QueryInterruptedException; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.query.QueryWatcher; import io.druid.query.metadata.metadata.ColumnAnalysis; import io.druid.query.metadata.metadata.ColumnIncluderator; import io.druid.query.metadata.metadata.SegmentAnalysis; @@ -40,16 +46,27 @@ import io.druid.segment.Segment; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory { private static final SegmentAnalyzer analyzer = new SegmentAnalyzer(); - private static final SegmentMetadataQueryQueryToolChest toolChest = new SegmentMetadataQueryQueryToolChest(); + private static final Logger log = new Logger(SegmentMetadataQueryRunnerFactory.class); + + private final QueryWatcher queryWatcher; + + @Inject + public SegmentMetadataQueryRunnerFactory( + QueryWatcher queryWatcher + ) + { + this.queryWatcher = queryWatcher; + } @Override public QueryRunner createRunner(final Segment segment) @@ -101,9 +118,10 @@ public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory mergeRunners( - final ExecutorService queryExecutor, Iterable> queryRunners + ExecutorService exec, Iterable> queryRunners ) { + final ListeningExecutorService queryExecutor = MoreExecutors.listeningDecorator(exec); return new ConcatQueryRunner( Sequences.map( Sequences.simple(queryRunners), @@ -118,7 +136,7 @@ public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory run(final Query query) { final int priority = query.getContextPriority(0); - Future> future = queryExecutor.submit( + ListenableFuture> future = queryExecutor.submit( new AbstractPrioritizedCallable>(priority) { @Override @@ -129,13 +147,25 @@ public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory getSegmentAnalysises(Segment index) { final QueryRunner runner = QueryRunnerTestHelper.makeQueryRunner( - (QueryRunnerFactory) new SegmentMetadataQueryRunnerFactory(), index + (QueryRunnerFactory) new SegmentMetadataQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER), index ); final SegmentMetadataQuery query = new SegmentMetadataQuery( diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index 5d596292348..ed1740460f8 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -45,7 +45,7 @@ public class SegmentMetadataQueryTest { @SuppressWarnings("unchecked") private final QueryRunner runner = makeQueryRunner( - new SegmentMetadataQueryRunnerFactory() + new SegmentMetadataQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER) ); private ObjectMapper mapper = new DefaultObjectMapper(); From ababbcadfdca77720717265979cca6df1280ea95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 10 Jun 2014 15:01:44 -0700 Subject: [PATCH 39/39] fix package groupId --- pom.xml | 2 +- server/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index fcc95f6d28f..a78251143a5 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ 1 - org.glassfish.web + org.glassfish javax.el 3.0.0 diff --git a/server/pom.xml b/server/pom.xml index f01e920cb77..de7da66f3ac 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -59,7 +59,7 @@ javax.inject - org.glassfish.web + org.glassfish javax.el