From 84f434d9f44eb279e3669af294c11bf685877afd Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 31 Jan 2014 16:32:21 -0800 Subject: [PATCH 01/17] [maven-release-plugin] prepare release druid-0.6.53 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 5 ++--- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 17 insertions(+), 18 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 0d33b40c711..15a008efdda 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/common/pom.xml b/common/pom.xml index b02af58de22..eafd46006d2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/examples/pom.xml b/examples/pom.xml index 1f77718f582..1d3c8716dcb 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 8ec3b6dcf4b..cfbb87b221f 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/hll/pom.xml b/hll/pom.xml index 58e17295c96..b6820ab5689 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -18,8 +18,7 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 io.druid.extensions druid-hll @@ -29,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 726a106e47c..d1117f90548 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 1a7881a56ff..27259f4b38d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 5ebec47b519..5a8f230b4db 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 6a26af7e73a..cfe96b1413e 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/pom.xml b/pom.xml index f18a852268c..b6a11c7650e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.53-SNAPSHOT + 0.6.53 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.53 diff --git a/processing/pom.xml b/processing/pom.xml index e12371ac4c1..80fce999a38 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 8779721fb40..452300afea2 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 6c8bfd24d6a..1001ce9eb5a 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/server/pom.xml b/server/pom.xml index d8bcfcd82f1..1e55b453cdb 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 diff --git a/services/pom.xml b/services/pom.xml index 2bb753c56bd..ee3816c8080 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.53-SNAPSHOT + 0.6.53 From 21d518e15c13616a7ced85ef9a5a50ae5c2604d1 Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 31 Jan 2014 16:32:25 -0800 Subject: [PATCH 02/17] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 15a008efdda..71fe0270c55 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index eafd46006d2..d33b11bc2b7 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 1d3c8716dcb..cb8c2bce6da 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index cfbb87b221f..7b2bdf00e16 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index b6820ab5689..f7d07eeeb49 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index d1117f90548..e82fea270b1 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 27259f4b38d..c7818874001 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 5a8f230b4db..bb872066635 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index cfe96b1413e..c4391660c05 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/pom.xml b/pom.xml index b6a11c7650e..8e4994a5f67 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.53 + 0.6.54-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.53 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index 80fce999a38..dd86b28c470 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 452300afea2..2eb140df098 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 1001ce9eb5a..b359dc0f0d4 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 1e55b453cdb..13c4fee7e9f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index ee3816c8080..365f7ed0224 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.53 + 0.6.54-SNAPSHOT From 5a152f8b1fd845929571abe99ef5bc3a3485e51a Mon Sep 17 00:00:00 2001 From: Mike Vincent Date: Sun, 2 Feb 2014 19:47:55 -0600 Subject: [PATCH 03/17] Link Data Formats document and format spec references --- docs/content/Data_formats.md | 13 +++++++------ docs/content/toc.textile | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/content/Data_formats.md b/docs/content/Data_formats.md index da02289e80c..1a6c5b3393d 100644 --- a/docs/content/Data_formats.md +++ b/docs/content/Data_formats.md @@ -1,8 +1,9 @@ --- layout: doc_page --- +Data Formats for Ingestion +========================== -# Data Formats for Ingestion Druid can ingest data in JSON, CSV, or TSV. While most examples in the documentation use data in JSON format, it is not difficult to configure Druid to ingest CSV or TSV data. ## Formatting the Data @@ -10,7 +11,7 @@ The following are three samples of the data used in the [Wikipedia example](Tuto _JSON_ -``` +```json {"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} {"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} {"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} @@ -43,7 +44,7 @@ Note that the CSV and TSV data do not contain column heads. This becomes importa ## Configuring Ingestion For the Indexing Service If you use the [indexing service](Indexing-Service.html) for ingesting the data, a [task](Tasks.html) must be configured and submitted. Tasks are configured with a JSON object which, among other things, specifies the data source and type. In the Wikipedia example, JSON data was read from a local file. The task spec contains a firehose element to specify this: - ... +```json "firehose" : { "type" : "local", "baseDir" : "examples/indexing", @@ -58,13 +59,13 @@ If you use the [indexing service](Indexing-Service.html) for ingesting the data, } } } - ... +``` Specified here are the location of the datafile, the timestamp column, the format of the data, and the columns that will become dimensions in Druid. Since the CSV data does not contain the column names, they will have to be added before that data can be processed: - ... +```json "firehose" : { "type" : "local", "baseDir" : "examples/indexing/", @@ -80,7 +81,7 @@ Since the CSV data does not contain the column names, they will have to be added } } } - ... +``` Note also that the filename extension and the data type were changed to "csv". For the TSV data, the same changes are made but with "tsv" for the filename extension and the data type. diff --git a/docs/content/toc.textile b/docs/content/toc.textile index e430534c42d..7f2e0eefe85 100644 --- a/docs/content/toc.textile +++ b/docs/content/toc.textile @@ -28,6 +28,7 @@ h2. Data Ingestion * "Batch":./Batch-ingestion.html * "Indexing Service":./Indexing-Service.html ** "Tasks":./Tasks.html +* "Data Formats":./Data_formats.html * "Ingestion FAQ":./Ingestion-FAQ.html h2. Querying From 994c7e3fa889a98edbbe586ed27306f76eb80688 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 3 Feb 2014 08:16:38 -0800 Subject: [PATCH 04/17] RemoteTaskActionClient: Retry on ChannelExceptions too --- .../druid/indexing/common/actions/RemoteTaskActionClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java index 175bc657e0d..915fa5a004f 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java @@ -32,6 +32,7 @@ import io.druid.curator.discovery.ServerDiscoverySelector; import io.druid.indexing.common.RetryPolicy; import io.druid.indexing.common.RetryPolicyFactory; import io.druid.indexing.common.task.Task; +import org.jboss.netty.channel.ChannelException; import org.joda.time.Duration; import java.io.IOException; @@ -94,6 +95,7 @@ public class RemoteTaskActionClient implements TaskActionClient } catch (Exception e) { Throwables.propagateIfInstanceOf(e.getCause(), IOException.class); + Throwables.propagateIfInstanceOf(e.getCause(), ChannelException.class); throw Throwables.propagate(e); } @@ -105,7 +107,7 @@ public class RemoteTaskActionClient implements TaskActionClient return jsonMapper.convertValue(responseDict.get("result"), taskAction.getReturnTypeReference()); } - catch (IOException e) { + catch (IOException | ChannelException e) { log.warn(e, "Exception submitting action for task[%s]", task.getId()); final Duration delay = retryPolicy.getAndIncrementRetryDelay(); From d4eba497224f3744b04ef07d09e0592ede8160fa Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:16:46 -0800 Subject: [PATCH 05/17] remove debug printing --- server/src/main/resources/static/js/handlers-0.0.2.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/resources/static/js/handlers-0.0.2.js b/server/src/main/resources/static/js/handlers-0.0.2.js index 8441fe9c561..139b1447dbc 100644 --- a/server/src/main/resources/static/js/handlers-0.0.2.js +++ b/server/src/main/resources/static/js/handlers-0.0.2.js @@ -8,7 +8,6 @@ $(document).ready(function() { function handleTable(dontDisplay) { - console.log(type); $.get(basePath + type + '?full', function(data) { buildTable(data, $('#result_table'), dontDisplay); @@ -76,7 +75,6 @@ $(document).ready(function() { } $('#view_button').click(function() { - console.log("here"); type = $('#select_type').val() + ''; view = $('#select_view').val() + ''; From f4b616530742cdd52bb13defc385212c2bed4675 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:33:06 -0800 Subject: [PATCH 06/17] fix bugs with finding indexing workers to assign --- server/src/main/java/io/druid/client/DruidServer.java | 5 +++++ .../main/java/io/druid/server/bridge/DruidClusterBridge.java | 2 +- .../java/io/druid/server/coordinator/DruidCoordinator.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java index 9cf0d992bd6..3a074f9596a 100644 --- a/server/src/main/java/io/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -136,6 +136,11 @@ public class DruidServer implements Comparable return Collections.unmodifiableMap(segments); } + public boolean isAssignable() + { + return getType().equalsIgnoreCase("historical") || getType().equalsIgnoreCase("bridge"); + } + public boolean isRealtime() { return getType().equalsIgnoreCase("realtime"); diff --git a/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java b/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java index f4073a18678..a029fe855f5 100644 --- a/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java +++ b/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java @@ -291,7 +291,7 @@ public class DruidClusterBridge DruidServer input ) { - return !input.isRealtime(); + return input.isAssignable(); } } ); diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index b11d2758035..78186119ee0 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -777,7 +777,7 @@ public class DruidCoordinator DruidServer input ) { - return !input.isRealtime(); + return input.isAssignable(); } } ); From cebb46a259cbbd3669eea919e69a28cc23fbf63f Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:34:57 -0800 Subject: [PATCH 07/17] [maven-release-plugin] prepare release druid-0.6.54 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 71fe0270c55..8d9d9537230 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/common/pom.xml b/common/pom.xml index d33b11bc2b7..e1d6cbf50a2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/examples/pom.xml b/examples/pom.xml index cb8c2bce6da..50a961d0e09 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 7b2bdf00e16..6970c13b024 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/hll/pom.xml b/hll/pom.xml index f7d07eeeb49..c1ef56fd784 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e82fea270b1..d1ba26463e2 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index c7818874001..dafb77b3f43 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index bb872066635..14b8e3b4f12 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index c4391660c05..f444ffc4c0e 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/pom.xml b/pom.xml index 8e4994a5f67..057004cc901 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.54-SNAPSHOT + 0.6.54 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.54 diff --git a/processing/pom.xml b/processing/pom.xml index dd86b28c470..6c957bef097 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 2eb140df098..c69d9b07bb1 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index b359dc0f0d4..4269259f5b8 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/server/pom.xml b/server/pom.xml index 13c4fee7e9f..e56f71af013 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/services/pom.xml b/services/pom.xml index 365f7ed0224..c15ab5e5f53 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 From 46571ddf6930425a944050e5a7cc5d6c7efa1ef7 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:35:02 -0800 Subject: [PATCH 08/17] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 8d9d9537230..5187619b531 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index e1d6cbf50a2..216d870b8f4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 50a961d0e09..c83222f1d63 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 6970c13b024..b9655bb295d 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index c1ef56fd784..37c9ceee8af 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index d1ba26463e2..5ac5626963b 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index dafb77b3f43..fa5eeaf237d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 14b8e3b4f12..e61ab45b604 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index f444ffc4c0e..07c6c899466 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/pom.xml b/pom.xml index 057004cc901..b2bf9e841e8 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.54 + 0.6.55-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.54 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index 6c957bef097..4b9e6e49f4b 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index c69d9b07bb1..45f32a65075 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 4269259f5b8..8c993151aed 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index e56f71af013..47108f8ed41 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index c15ab5e5f53..37e79659eb2 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT From 8b398533cb0262b40b73370cdfbe6f90933415b6 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:45:59 -0800 Subject: [PATCH 09/17] fix more bugs with determining what is assignable and what is not --- .../main/java/io/druid/client/CachingClusteredClient.java | 2 +- server/src/main/java/io/druid/client/DruidServer.java | 5 ----- .../io/druid/segment/realtime/plumber/RealtimePlumber.java | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index f5b868e30fb..dacfc7938ef 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -313,7 +313,7 @@ public class CachingClusteredClient implements QueryRunner final MultipleSpecificSegmentSpec segmentSpec = new MultipleSpecificSegmentSpec(descriptors); List intervals = segmentSpec.getIntervals(); - if (server.isRealtime() || !populateCache || isBySegment) { + if (!server.isAssignable() || !populateCache || isBySegment) { resultSeqToAdd = clientQueryable.run(query.withQuerySegmentSpec(segmentSpec)); } else { resultSeqToAdd = toolChest.mergeSequences( diff --git a/server/src/main/java/io/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java index 3a074f9596a..c33b9270b57 100644 --- a/server/src/main/java/io/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -141,11 +141,6 @@ public class DruidServer implements Comparable return getType().equalsIgnoreCase("historical") || getType().equalsIgnoreCase("bridge"); } - public boolean isRealtime() - { - return getType().equalsIgnoreCase("realtime"); - } - public DataSegment getSegment(String segmentName) { return segments.get(segmentName); diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 2b4d1f5f44b..62b3d3d5f6d 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -704,7 +704,7 @@ public class RealtimePlumber implements Plumber return ServerView.CallbackAction.UNREGISTER; } - if (server.isRealtime()) { + if (!server.isAssignable()) { return ServerView.CallbackAction.CONTINUE; } From 4d52e20fe17195634e5fab2fe4b34a96701c16c9 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:47:46 -0800 Subject: [PATCH 10/17] [maven-release-plugin] prepare release druid-0.6.55 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 5187619b531..3ee42a1fe82 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/common/pom.xml b/common/pom.xml index 216d870b8f4..38e3fc8481e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/examples/pom.xml b/examples/pom.xml index c83222f1d63..c11cb2f3de2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index b9655bb295d..c0a21234f92 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/hll/pom.xml b/hll/pom.xml index 37c9ceee8af..57c55f4fec6 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 5ac5626963b..980791a0e64 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index fa5eeaf237d..ec59d7aa318 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index e61ab45b604..eeb5878d2b4 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 07c6c899466..df4eea682e2 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/pom.xml b/pom.xml index b2bf9e841e8..51c6e9f304c 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.55-SNAPSHOT + 0.6.55 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.55 diff --git a/processing/pom.xml b/processing/pom.xml index 4b9e6e49f4b..f91058a6c56 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 45f32a65075..0d20b0ce879 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 8c993151aed..ce1239b7119 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/server/pom.xml b/server/pom.xml index 47108f8ed41..888b368408b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/services/pom.xml b/services/pom.xml index 37e79659eb2..f2640eeb5b8 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 From fc4cd79125f0b0b12697fbe594f68324a3bb8122 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:47:50 -0800 Subject: [PATCH 11/17] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 3ee42a1fe82..d35dcd94e91 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 38e3fc8481e..8877fb3f5f9 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index c11cb2f3de2..d464bfd1348 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index c0a21234f92..203be72fa67 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index 57c55f4fec6..c90622c740a 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 980791a0e64..18affe138a1 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index ec59d7aa318..5dbc2be0d9d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index eeb5878d2b4..99fce7e8869 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index df4eea682e2..a41bf9b0ab2 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/pom.xml b/pom.xml index 51c6e9f304c..51969af12c6 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.55 + 0.6.56-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.55 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index f91058a6c56..a6add656a6b 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 0d20b0ce879..f255d6fdd00 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index ce1239b7119..d4eaabe9eac 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 888b368408b..269bcd1b97a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index f2640eeb5b8..f443be47515 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT From badc7b2e3fde37e57f4009f2b17135ff1fb7968c Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:21:45 -0800 Subject: [PATCH 12/17] fix bug with load throttling and more tests --- .../coordinator/SegmentReplicantLookup.java | 37 ++++++-- .../server/coordinator/rules/LoadRule.java | 71 ++++++++------- .../DruidCoordinatorRuleRunnerTest.java | 88 +++++++++++++++++++ .../coordinator/rules/LoadRuleTest.java | 2 +- 4 files changed, 160 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java index a90eeba4cae..fddaf0d7c05 100644 --- a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java +++ b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java @@ -83,12 +83,22 @@ public class SegmentReplicantLookup } public Map getLoadingTiers(String segmentId) - { - Map retVal = loadingSegments.row(segmentId); - return (retVal == null) ? Maps.newHashMap() : retVal; - } + { + Map retVal = loadingSegments.row(segmentId); + return (retVal == null) ? Maps.newHashMap() : retVal; + } - public int getClusterReplicants(String segmentId, String tier) + public int getLoadedReplicants(String segmentId) + { + Map allTiers = segmentsInCluster.row(segmentId); + int retVal = 0; + for (Integer replicants : allTiers.values()) { + retVal += replicants; + } + return retVal; + } + + public int getLoadedReplicants(String segmentId, String tier) { Integer retVal = segmentsInCluster.get(segmentId, tier); return (retVal == null) ? 0 : retVal; @@ -100,8 +110,23 @@ public class SegmentReplicantLookup return (retVal == null) ? 0 : retVal; } + public int getLoadingReplicants(String segmentId) + { + Map allTiers = loadingSegments.row(segmentId); + int retVal = 0; + for (Integer replicants : allTiers.values()) { + retVal += replicants; + } + return retVal; + } + + public int getTotalReplicants(String segmentId) + { + return getLoadedReplicants(segmentId) + getLoadingReplicants(segmentId); + } + public int getTotalReplicants(String segmentId, String tier) { - return getClusterReplicants(segmentId, tier) + getLoadingReplicants(segmentId, tier); + return getLoadedReplicants(segmentId, tier) + getLoadingReplicants(segmentId, tier); } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java index f629318395b..cac7a5b8c28 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java @@ -42,6 +42,8 @@ import java.util.Map; public abstract class LoadRule implements Rule { private static final EmittingLogger log = new EmittingLogger(LoadRule.class); + private static final String assignedCount = "assignedCount"; + private static final String droppedCount = "droppedCount"; @Override public CoordinatorStats run(DruidCoordinator coordinator, DruidCoordinatorRuntimeParams params, DataSegment segment) @@ -49,13 +51,17 @@ public abstract class LoadRule implements Rule CoordinatorStats stats = new CoordinatorStats(); final Map loadStatus = Maps.newHashMap(); + + int totalReplicantsInCluster = params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier()); for (Map.Entry entry : getTieredReplicants().entrySet()) { final String tier = entry.getKey(); - final int expectedReplicants = entry.getValue(); + final int expectedReplicantsInTier = entry.getValue(); + final int totalReplicantsInTier = params.getSegmentReplicantLookup() + .getTotalReplicants(segment.getIdentifier(), tier); + final int loadedReplicantsInTier = params.getSegmentReplicantLookup() + .getLoadedReplicants(segment.getIdentifier(), tier); - int totalReplicants = params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier(), tier); - - MinMaxPriorityQueue serverQueue = params.getDruidCluster().getServersByTier(tier); + final MinMaxPriorityQueue serverQueue = params.getDruidCluster().getServersByTier(tier); if (serverQueue == null) { log.makeAlert("Tier[%s] has no servers! Check your cluster configuration!", tier).emit(); return stats; @@ -65,22 +71,21 @@ public abstract class LoadRule implements Rule final DateTime referenceTimestamp = params.getBalancerReferenceTimestamp(); final BalancerStrategy strategy = params.getBalancerStrategyFactory().createBalancerStrategy(referenceTimestamp); if (params.getAvailableSegments().contains(segment)) { - stats.accumulate( - assign( - params.getReplicationManager(), - tier, - expectedReplicants, - totalReplicants, - strategy, - serverHolderList, - segment - ) + CoordinatorStats assignStats = assign( + params.getReplicationManager(), + tier, + totalReplicantsInCluster, + expectedReplicantsInTier, + totalReplicantsInTier, + strategy, + serverHolderList, + segment ); + stats.accumulate(assignStats); + totalReplicantsInCluster += assignStats.getPerTierStats().get(assignedCount).get(tier).get(); } - int clusterReplicants = params.getSegmentReplicantLookup() - .getClusterReplicants(segment.getIdentifier(), tier); - loadStatus.put(tier, expectedReplicants - clusterReplicants); + loadStatus.put(tier, expectedReplicantsInTier - loadedReplicantsInTier); } // Remove over-replication stats.accumulate(drop(loadStatus, segment, params)); @@ -92,18 +97,21 @@ public abstract class LoadRule implements Rule private CoordinatorStats assign( final ReplicationThrottler replicationManager, final String tier, - final int expectedReplicants, - int totalReplicants, + final int totalReplicantsInCluster, + final int expectedReplicantsInTier, + final int totalReplicantsInTier, final BalancerStrategy strategy, final List serverHolderList, final DataSegment segment ) { final CoordinatorStats stats = new CoordinatorStats(); - stats.addToTieredStat("assignedCount", tier, 0); + stats.addToTieredStat(assignedCount, tier, 0); - while (totalReplicants < expectedReplicants) { - boolean replicate = totalReplicants > 0; + int currReplicantsInTier = totalReplicantsInTier; + int currTotalReplicantsInCluster = totalReplicantsInCluster; + while (currReplicantsInTier < expectedReplicantsInTier) { + boolean replicate = currTotalReplicantsInCluster > 0; if (replicate && !replicationManager.canCreateReplicant(tier)) { break; @@ -116,7 +124,7 @@ public abstract class LoadRule implements Rule "Not enough [%s] servers or node capacity to assign segment[%s]! Expected Replicants[%d]", tier, segment.getIdentifier(), - expectedReplicants + expectedReplicantsInTier ); break; } @@ -143,8 +151,9 @@ public abstract class LoadRule implements Rule } ); - stats.addToTieredStat("assignedCount", tier, 1); - ++totalReplicants; + stats.addToTieredStat(assignedCount, tier, 1); + ++currReplicantsInTier; + ++currTotalReplicantsInCluster; } return stats; @@ -162,7 +171,7 @@ public abstract class LoadRule implements Rule return stats; } - // Make sure we have enough actual replicants in the correct tiers in the cluster before doing anything + // Make sure we have enough loaded replicants in the correct tiers in the cluster before doing anything for (Integer leftToLoad : loadStatus.values()) { if (leftToLoad > 0) { return stats; @@ -176,10 +185,10 @@ public abstract class LoadRule implements Rule for (Map.Entry entry : replicantsByTier.entrySet()) { final String tier = entry.getKey(); - int actualNumReplicantsForTier = entry.getValue(); + int loadedNumReplicantsForTier = entry.getValue(); int expectedNumReplicantsForTier = getNumReplicants(tier); - stats.addToTieredStat("droppedCount", tier, 0); + stats.addToTieredStat(droppedCount, tier, 0); MinMaxPriorityQueue serverQueue = params.getDruidCluster().get(tier); if (serverQueue == null) { @@ -188,7 +197,7 @@ public abstract class LoadRule implements Rule } List droppedServers = Lists.newArrayList(); - while (actualNumReplicantsForTier > expectedNumReplicantsForTier) { + while (loadedNumReplicantsForTier > expectedNumReplicantsForTier) { final ServerHolder holder = serverQueue.pollLast(); if (holder == null) { log.warn("Wtf, holder was null? I have no servers serving [%s]?", segment.getIdentifier()); @@ -224,8 +233,8 @@ public abstract class LoadRule implements Rule } } ); - --actualNumReplicantsForTier; - stats.addToTieredStat("droppedCount", tier, 1); + --loadedNumReplicantsForTier; + stats.addToTieredStat(droppedCount, tier, 1); } droppedServers.add(holder); } diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java index 2663a4a94e4..95d52c1fad1 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java @@ -979,6 +979,94 @@ public class DruidCoordinatorRuleRunnerTest EasyMock.verify(mockPeon); } + /** + * Nodes: + * hot - nothing loaded + * _default_tier - 1 segment loaded + * + * @throws Exception + */ + @Test + public void testReplicantThrottleAcrossTiers() throws Exception + { + mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject()); + EasyMock.expectLastCall().atLeastOnce(); + EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.newHashSet()).atLeastOnce(); + EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce(); + EasyMock.replay(mockPeon); + + EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn( + Lists.newArrayList( + new IntervalLoadRule( + new Interval("2012-01-01T00:00:00.000Z/2013-01-01T00:00:00.000Z"), + ImmutableMap.of( + "hot", 1, + DruidServer.DEFAULT_TIER, 1 + ), + null, + null + ) + ) + ).atLeastOnce(); + EasyMock.replay(databaseRuleManager); + + DruidCluster druidCluster = new DruidCluster( + ImmutableMap.of( + "hot", + MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create( + Arrays.asList( + new ServerHolder( + new DruidServer( + "serverHot", + "hostHot", + 1000, + "historical", + "hot", + 0 + ), + mockPeon + ) + ) + ), + DruidServer.DEFAULT_TIER, + MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create( + Arrays.asList( + new ServerHolder( + new DruidServer( + "serverNorm", + "hostNorm", + 1000, + "historical", + DruidServer.DEFAULT_TIER, + 0 + ), + mockPeon + ) + ) + ) + ) + ); + + DruidCoordinatorRuntimeParams params = + new DruidCoordinatorRuntimeParams.Builder() + .withDruidCluster(druidCluster) + .withAvailableSegments(availableSegments) + .withDatabaseRuleManager(databaseRuleManager) + .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) + .build(); + + DruidCoordinatorRuleRunner runner = new DruidCoordinatorRuleRunner(new ReplicationThrottler(7, 1), coordinator); + DruidCoordinatorRuntimeParams afterParams = runner.run(params); + CoordinatorStats stats = afterParams.getCoordinatorStats(); + + Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get("hot").get() == 24); + Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get(DruidServer.DEFAULT_TIER).get() == 7); + Assert.assertTrue(stats.getPerTierStats().get("unassignedCount") == null); + Assert.assertTrue(stats.getPerTierStats().get("unassignedSize") == null); + + EasyMock.verify(mockPeon); + } + @Test public void testDropReplicantThrottle() throws Exception { diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index a52927d0b7d..3e75d1b79c3 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -59,7 +59,7 @@ public class LoadRuleTest public void setUp() throws Exception { mockPeon = EasyMock.createMock(LoadQueuePeon.class); - throttler = new ReplicationThrottler(1, 1); + throttler = new ReplicationThrottler(2, 1); for (String tier : Arrays.asList("hot", DruidServer.DEFAULT_TIER)) { throttler.updateReplicationState(tier); throttler.updateTerminationState(tier); From 0ea45b590feb4f0c1c9089e1d0561e2cb6f9a09c Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:23:30 -0800 Subject: [PATCH 13/17] [maven-release-plugin] prepare release druid-0.6.56 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index d35dcd94e91..c8f55e15a03 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/common/pom.xml b/common/pom.xml index 8877fb3f5f9..780138b3910 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/examples/pom.xml b/examples/pom.xml index d464bfd1348..4ec36caf36b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 203be72fa67..8965bc65a31 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/hll/pom.xml b/hll/pom.xml index c90622c740a..8b51cc4e0b3 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 18affe138a1..46514b00e93 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 5dbc2be0d9d..71110b5937d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 99fce7e8869..1101d19c292 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index a41bf9b0ab2..20bdc7e8ccd 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/pom.xml b/pom.xml index 51969af12c6..cb461a668e7 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.56-SNAPSHOT + 0.6.56 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.56 diff --git a/processing/pom.xml b/processing/pom.xml index a6add656a6b..761c92f0ac7 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index f255d6fdd00..49b2d1c4bed 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index d4eaabe9eac..b07cc632bc3 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/server/pom.xml b/server/pom.xml index 269bcd1b97a..412ee9bae53 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/services/pom.xml b/services/pom.xml index f443be47515..64712465d8c 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 From 056e82a51684a61b9faf07eee042007153b3f5db Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:23:34 -0800 Subject: [PATCH 14/17] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index c8f55e15a03..1172adf7564 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 780138b3910..497216bb8f6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 4ec36caf36b..1c4451a81e8 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 8965bc65a31..7f8bdae4830 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index 8b51cc4e0b3..e566b93bb84 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 46514b00e93..6a960315b5f 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 71110b5937d..128c692d459 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 1101d19c292..a049ab9cb47 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 20bdc7e8ccd..d8af1ffe3a5 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/pom.xml b/pom.xml index cb461a668e7..eed8fa2bd0d 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.56 + 0.6.57-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.56 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index 761c92f0ac7..e9f51ce1228 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 49b2d1c4bed..c407d4387d0 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index b07cc632bc3..6170c861d14 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 412ee9bae53..a6bf41d1f36 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 64712465d8c..1962aee650e 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT From 53c5124b992d40e67e779787097650ef42a0b6f7 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 5 Feb 2014 15:59:00 -0800 Subject: [PATCH 15/17] DbConnector: Indexes on tasks table --- common/src/main/java/io/druid/db/DbConnector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/io/druid/db/DbConnector.java b/common/src/main/java/io/druid/db/DbConnector.java index 03b8987297b..585276ade91 100644 --- a/common/src/main/java/io/druid/db/DbConnector.java +++ b/common/src/main/java/io/druid/db/DbConnector.java @@ -87,7 +87,8 @@ public class DbConnector + " `payload` longblob NOT NULL,\n" + " `status_payload` longblob NOT NULL,\n" + " `active` tinyint(1) NOT NULL DEFAULT '0',\n" - + " PRIMARY KEY (`id`)\n" + + " PRIMARY KEY (`id`),\n" + + " KEY (active, created_date(100))\n" + ")", taskTableName ) From af4827336939096ea179d60788d8e3dfa54af569 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 6 Feb 2014 11:19:05 -0800 Subject: [PATCH 16/17] fix bug with dynamic configs in coordinator not working --- .../overlord/http/OverlordResource.java | 1 - .../server/coordinator/DruidCoordinator.java | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java index 5a3eb1c5e07..58c917c0a26 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java @@ -46,7 +46,6 @@ import io.druid.indexing.overlord.scaling.ResourceManagementScheduler; import io.druid.indexing.overlord.setup.WorkerSetupData; import io.druid.tasklogs.TaskLogStreamer; import io.druid.timeline.DataSegment; -import org.apache.zookeeper.data.Stat; import org.joda.time.DateTime; import javax.ws.rs.Consumes; diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 78186119ee0..f5f202b5904 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -96,8 +96,8 @@ public class DruidCoordinator private final Object lock = new Object(); private volatile boolean started = false; + private volatile int leaderCounter = 0; private volatile boolean leader = false; - private volatile AtomicReference dynamicConfigs; private final DruidCoordinatorConfig config; private final ZkPathsConfig zkPaths; @@ -184,7 +184,6 @@ public class DruidCoordinator this.exec = scheduledExecutorFactory.create(1, "Coordinator-Exec--%d"); this.leaderLatch = new AtomicReference<>(null); - this.dynamicConfigs = new AtomicReference<>(null); this.loadManagementPeons = loadQueuePeonMap; } @@ -280,7 +279,11 @@ public class DruidCoordinator public CoordinatorDynamicConfig getDynamicConfigs() { - return dynamicConfigs.get(); + return configManager.watch( + CoordinatorDynamicConfig.CONFIG_KEY, + CoordinatorDynamicConfig.class, + new CoordinatorDynamicConfig.Builder().build() + ).get(); } public void removeSegment(DataSegment segment) @@ -533,6 +536,7 @@ public class DruidCoordinator log.info("I am the leader of the coordinators, all must bow!"); try { + leaderCounter++; leader = true; databaseSegmentManager.start(); databaseRuleManager.start(); @@ -540,11 +544,6 @@ public class DruidCoordinator serviceAnnouncer.announce(self); final List> coordinatorRunnables = Lists.newArrayList(); - dynamicConfigs = configManager.watch( - CoordinatorDynamicConfig.CONFIG_KEY, - CoordinatorDynamicConfig.class, - new CoordinatorDynamicConfig.Builder().build() - ); coordinatorRunnables.add(Pair.of(new CoordinatorHistoricalManagerRunnable(), config.getCoordinatorPeriod())); if (indexingServiceClient != null) { coordinatorRunnables.add( @@ -562,6 +561,7 @@ public class DruidCoordinator ); } + final int startingLeaderCounter = leaderCounter; for (final Pair coordinatorRunnable : coordinatorRunnables) { ScheduledExecutors.scheduleWithFixedDelay( exec, @@ -574,10 +574,10 @@ public class DruidCoordinator @Override public ScheduledExecutors.Signal call() { - if (leader) { + if (leader && startingLeaderCounter == leaderCounter) { theRunnable.run(); } - if (leader) { // (We might no longer be leader) + if (leader && startingLeaderCounter == leaderCounter) { // (We might no longer be leader) return ScheduledExecutors.Signal.REPEAT; } else { return ScheduledExecutors.Signal.STOP; @@ -610,6 +610,8 @@ public class DruidCoordinator { synchronized (lock) { try { + leaderCounter++; + log.info("I am no longer the leader..."); for (String server : loadManagementPeons.keySet()) { @@ -739,11 +741,10 @@ public class DruidCoordinator DruidCoordinatorRuntimeParams.newBuilder() .withStartTime(startTime) .withDatasources(databaseSegmentManager.getInventory()) - .withDynamicConfigs(dynamicConfigs.get()) + .withDynamicConfigs(getDynamicConfigs()) .withEmitter(emitter) .build(); - for (DruidCoordinatorHelper helper : helpers) { params = helper.run(params); } @@ -826,7 +827,6 @@ public class DruidCoordinator .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) - .withDynamicConfigs(dynamicConfigs.get()) .build(); } }, From 9756b18f9b5cf4f32438d42590d9d4e0fac47523 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 6 Feb 2014 11:26:05 -0800 Subject: [PATCH 17/17] sort complex metrics with groupby --- .../java/io/druid/query/groupby/orderby/DefaultLimitSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index 4cd052058bf..3bb79ed9617 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -148,7 +148,7 @@ public class DefaultLimitSpec implements LimitSpec @Override public int compare(Row left, Row right) { - return comparator.compare(left.getFloatMetric(column), right.getFloatMetric(column)); + return comparator.compare(left.getRaw(column), right.getRaw(column)); } }; }