From d4eba497224f3744b04ef07d09e0592ede8160fa Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:16:46 -0800 Subject: [PATCH 01/13] remove debug printing --- server/src/main/resources/static/js/handlers-0.0.2.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/resources/static/js/handlers-0.0.2.js b/server/src/main/resources/static/js/handlers-0.0.2.js index 8441fe9c561..139b1447dbc 100644 --- a/server/src/main/resources/static/js/handlers-0.0.2.js +++ b/server/src/main/resources/static/js/handlers-0.0.2.js @@ -8,7 +8,6 @@ $(document).ready(function() { function handleTable(dontDisplay) { - console.log(type); $.get(basePath + type + '?full', function(data) { buildTable(data, $('#result_table'), dontDisplay); @@ -76,7 +75,6 @@ $(document).ready(function() { } $('#view_button').click(function() { - console.log("here"); type = $('#select_type').val() + ''; view = $('#select_view').val() + ''; From f4b616530742cdd52bb13defc385212c2bed4675 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:33:06 -0800 Subject: [PATCH 02/13] fix bugs with finding indexing workers to assign --- server/src/main/java/io/druid/client/DruidServer.java | 5 +++++ .../main/java/io/druid/server/bridge/DruidClusterBridge.java | 2 +- .../java/io/druid/server/coordinator/DruidCoordinator.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java index 9cf0d992bd6..3a074f9596a 100644 --- a/server/src/main/java/io/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -136,6 +136,11 @@ public class DruidServer implements Comparable return Collections.unmodifiableMap(segments); } + public boolean isAssignable() + { + return getType().equalsIgnoreCase("historical") || getType().equalsIgnoreCase("bridge"); + } + public boolean isRealtime() { return getType().equalsIgnoreCase("realtime"); diff --git a/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java b/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java index f4073a18678..a029fe855f5 100644 --- a/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java +++ b/server/src/main/java/io/druid/server/bridge/DruidClusterBridge.java @@ -291,7 +291,7 @@ public class DruidClusterBridge DruidServer input ) { - return !input.isRealtime(); + return input.isAssignable(); } } ); diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index b11d2758035..78186119ee0 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -777,7 +777,7 @@ public class DruidCoordinator DruidServer input ) { - return !input.isRealtime(); + return input.isAssignable(); } } ); From cebb46a259cbbd3669eea919e69a28cc23fbf63f Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:34:57 -0800 Subject: [PATCH 03/13] [maven-release-plugin] prepare release druid-0.6.54 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 71fe0270c55..8d9d9537230 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/common/pom.xml b/common/pom.xml index d33b11bc2b7..e1d6cbf50a2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/examples/pom.xml b/examples/pom.xml index cb8c2bce6da..50a961d0e09 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 7b2bdf00e16..6970c13b024 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/hll/pom.xml b/hll/pom.xml index f7d07eeeb49..c1ef56fd784 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e82fea270b1..d1ba26463e2 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index c7818874001..dafb77b3f43 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index bb872066635..14b8e3b4f12 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index c4391660c05..f444ffc4c0e 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/pom.xml b/pom.xml index 8e4994a5f67..057004cc901 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.54-SNAPSHOT + 0.6.54 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.54 diff --git a/processing/pom.xml b/processing/pom.xml index dd86b28c470..6c957bef097 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 2eb140df098..c69d9b07bb1 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index b359dc0f0d4..4269259f5b8 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/server/pom.xml b/server/pom.xml index 13c4fee7e9f..e56f71af013 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 diff --git a/services/pom.xml b/services/pom.xml index 365f7ed0224..c15ab5e5f53 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.54-SNAPSHOT + 0.6.54 From 46571ddf6930425a944050e5a7cc5d6c7efa1ef7 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:35:02 -0800 Subject: [PATCH 04/13] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 8d9d9537230..5187619b531 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index e1d6cbf50a2..216d870b8f4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 50a961d0e09..c83222f1d63 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 6970c13b024..b9655bb295d 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index c1ef56fd784..37c9ceee8af 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index d1ba26463e2..5ac5626963b 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index dafb77b3f43..fa5eeaf237d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 14b8e3b4f12..e61ab45b604 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index f444ffc4c0e..07c6c899466 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/pom.xml b/pom.xml index 057004cc901..b2bf9e841e8 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.54 + 0.6.55-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.54 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index 6c957bef097..4b9e6e49f4b 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index c69d9b07bb1..45f32a65075 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 4269259f5b8..8c993151aed 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index e56f71af013..47108f8ed41 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index c15ab5e5f53..37e79659eb2 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.54 + 0.6.55-SNAPSHOT From 8b398533cb0262b40b73370cdfbe6f90933415b6 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:45:59 -0800 Subject: [PATCH 05/13] fix more bugs with determining what is assignable and what is not --- .../main/java/io/druid/client/CachingClusteredClient.java | 2 +- server/src/main/java/io/druid/client/DruidServer.java | 5 ----- .../io/druid/segment/realtime/plumber/RealtimePlumber.java | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index f5b868e30fb..dacfc7938ef 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -313,7 +313,7 @@ public class CachingClusteredClient implements QueryRunner final MultipleSpecificSegmentSpec segmentSpec = new MultipleSpecificSegmentSpec(descriptors); List intervals = segmentSpec.getIntervals(); - if (server.isRealtime() || !populateCache || isBySegment) { + if (!server.isAssignable() || !populateCache || isBySegment) { resultSeqToAdd = clientQueryable.run(query.withQuerySegmentSpec(segmentSpec)); } else { resultSeqToAdd = toolChest.mergeSequences( diff --git a/server/src/main/java/io/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java index 3a074f9596a..c33b9270b57 100644 --- a/server/src/main/java/io/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -141,11 +141,6 @@ public class DruidServer implements Comparable return getType().equalsIgnoreCase("historical") || getType().equalsIgnoreCase("bridge"); } - public boolean isRealtime() - { - return getType().equalsIgnoreCase("realtime"); - } - public DataSegment getSegment(String segmentName) { return segments.get(segmentName); diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 2b4d1f5f44b..62b3d3d5f6d 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -704,7 +704,7 @@ public class RealtimePlumber implements Plumber return ServerView.CallbackAction.UNREGISTER; } - if (server.isRealtime()) { + if (!server.isAssignable()) { return ServerView.CallbackAction.CONTINUE; } From 4d52e20fe17195634e5fab2fe4b34a96701c16c9 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:47:46 -0800 Subject: [PATCH 06/13] [maven-release-plugin] prepare release druid-0.6.55 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 5187619b531..3ee42a1fe82 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/common/pom.xml b/common/pom.xml index 216d870b8f4..38e3fc8481e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/examples/pom.xml b/examples/pom.xml index c83222f1d63..c11cb2f3de2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index b9655bb295d..c0a21234f92 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/hll/pom.xml b/hll/pom.xml index 37c9ceee8af..57c55f4fec6 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 5ac5626963b..980791a0e64 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index fa5eeaf237d..ec59d7aa318 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index e61ab45b604..eeb5878d2b4 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 07c6c899466..df4eea682e2 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/pom.xml b/pom.xml index b2bf9e841e8..51c6e9f304c 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.55-SNAPSHOT + 0.6.55 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.55 diff --git a/processing/pom.xml b/processing/pom.xml index 4b9e6e49f4b..f91058a6c56 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 45f32a65075..0d20b0ce879 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 8c993151aed..ce1239b7119 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/server/pom.xml b/server/pom.xml index 47108f8ed41..888b368408b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 diff --git a/services/pom.xml b/services/pom.xml index 37e79659eb2..f2640eeb5b8 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.55-SNAPSHOT + 0.6.55 From fc4cd79125f0b0b12697fbe594f68324a3bb8122 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 3 Feb 2014 16:47:50 -0800 Subject: [PATCH 07/13] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 3ee42a1fe82..d35dcd94e91 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 38e3fc8481e..8877fb3f5f9 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index c11cb2f3de2..d464bfd1348 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index c0a21234f92..203be72fa67 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index 57c55f4fec6..c90622c740a 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 980791a0e64..18affe138a1 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index ec59d7aa318..5dbc2be0d9d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index eeb5878d2b4..99fce7e8869 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index df4eea682e2..a41bf9b0ab2 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/pom.xml b/pom.xml index 51c6e9f304c..51969af12c6 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.55 + 0.6.56-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.55 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index f91058a6c56..a6add656a6b 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 0d20b0ce879..f255d6fdd00 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index ce1239b7119..d4eaabe9eac 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 888b368408b..269bcd1b97a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index f2640eeb5b8..f443be47515 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.55 + 0.6.56-SNAPSHOT From badc7b2e3fde37e57f4009f2b17135ff1fb7968c Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:21:45 -0800 Subject: [PATCH 08/13] fix bug with load throttling and more tests --- .../coordinator/SegmentReplicantLookup.java | 37 ++++++-- .../server/coordinator/rules/LoadRule.java | 71 ++++++++------- .../DruidCoordinatorRuleRunnerTest.java | 88 +++++++++++++++++++ .../coordinator/rules/LoadRuleTest.java | 2 +- 4 files changed, 160 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java index a90eeba4cae..fddaf0d7c05 100644 --- a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java +++ b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java @@ -83,12 +83,22 @@ public class SegmentReplicantLookup } public Map getLoadingTiers(String segmentId) - { - Map retVal = loadingSegments.row(segmentId); - return (retVal == null) ? Maps.newHashMap() : retVal; - } + { + Map retVal = loadingSegments.row(segmentId); + return (retVal == null) ? Maps.newHashMap() : retVal; + } - public int getClusterReplicants(String segmentId, String tier) + public int getLoadedReplicants(String segmentId) + { + Map allTiers = segmentsInCluster.row(segmentId); + int retVal = 0; + for (Integer replicants : allTiers.values()) { + retVal += replicants; + } + return retVal; + } + + public int getLoadedReplicants(String segmentId, String tier) { Integer retVal = segmentsInCluster.get(segmentId, tier); return (retVal == null) ? 0 : retVal; @@ -100,8 +110,23 @@ public class SegmentReplicantLookup return (retVal == null) ? 0 : retVal; } + public int getLoadingReplicants(String segmentId) + { + Map allTiers = loadingSegments.row(segmentId); + int retVal = 0; + for (Integer replicants : allTiers.values()) { + retVal += replicants; + } + return retVal; + } + + public int getTotalReplicants(String segmentId) + { + return getLoadedReplicants(segmentId) + getLoadingReplicants(segmentId); + } + public int getTotalReplicants(String segmentId, String tier) { - return getClusterReplicants(segmentId, tier) + getLoadingReplicants(segmentId, tier); + return getLoadedReplicants(segmentId, tier) + getLoadingReplicants(segmentId, tier); } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java index f629318395b..cac7a5b8c28 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java @@ -42,6 +42,8 @@ import java.util.Map; public abstract class LoadRule implements Rule { private static final EmittingLogger log = new EmittingLogger(LoadRule.class); + private static final String assignedCount = "assignedCount"; + private static final String droppedCount = "droppedCount"; @Override public CoordinatorStats run(DruidCoordinator coordinator, DruidCoordinatorRuntimeParams params, DataSegment segment) @@ -49,13 +51,17 @@ public abstract class LoadRule implements Rule CoordinatorStats stats = new CoordinatorStats(); final Map loadStatus = Maps.newHashMap(); + + int totalReplicantsInCluster = params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier()); for (Map.Entry entry : getTieredReplicants().entrySet()) { final String tier = entry.getKey(); - final int expectedReplicants = entry.getValue(); + final int expectedReplicantsInTier = entry.getValue(); + final int totalReplicantsInTier = params.getSegmentReplicantLookup() + .getTotalReplicants(segment.getIdentifier(), tier); + final int loadedReplicantsInTier = params.getSegmentReplicantLookup() + .getLoadedReplicants(segment.getIdentifier(), tier); - int totalReplicants = params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier(), tier); - - MinMaxPriorityQueue serverQueue = params.getDruidCluster().getServersByTier(tier); + final MinMaxPriorityQueue serverQueue = params.getDruidCluster().getServersByTier(tier); if (serverQueue == null) { log.makeAlert("Tier[%s] has no servers! Check your cluster configuration!", tier).emit(); return stats; @@ -65,22 +71,21 @@ public abstract class LoadRule implements Rule final DateTime referenceTimestamp = params.getBalancerReferenceTimestamp(); final BalancerStrategy strategy = params.getBalancerStrategyFactory().createBalancerStrategy(referenceTimestamp); if (params.getAvailableSegments().contains(segment)) { - stats.accumulate( - assign( - params.getReplicationManager(), - tier, - expectedReplicants, - totalReplicants, - strategy, - serverHolderList, - segment - ) + CoordinatorStats assignStats = assign( + params.getReplicationManager(), + tier, + totalReplicantsInCluster, + expectedReplicantsInTier, + totalReplicantsInTier, + strategy, + serverHolderList, + segment ); + stats.accumulate(assignStats); + totalReplicantsInCluster += assignStats.getPerTierStats().get(assignedCount).get(tier).get(); } - int clusterReplicants = params.getSegmentReplicantLookup() - .getClusterReplicants(segment.getIdentifier(), tier); - loadStatus.put(tier, expectedReplicants - clusterReplicants); + loadStatus.put(tier, expectedReplicantsInTier - loadedReplicantsInTier); } // Remove over-replication stats.accumulate(drop(loadStatus, segment, params)); @@ -92,18 +97,21 @@ public abstract class LoadRule implements Rule private CoordinatorStats assign( final ReplicationThrottler replicationManager, final String tier, - final int expectedReplicants, - int totalReplicants, + final int totalReplicantsInCluster, + final int expectedReplicantsInTier, + final int totalReplicantsInTier, final BalancerStrategy strategy, final List serverHolderList, final DataSegment segment ) { final CoordinatorStats stats = new CoordinatorStats(); - stats.addToTieredStat("assignedCount", tier, 0); + stats.addToTieredStat(assignedCount, tier, 0); - while (totalReplicants < expectedReplicants) { - boolean replicate = totalReplicants > 0; + int currReplicantsInTier = totalReplicantsInTier; + int currTotalReplicantsInCluster = totalReplicantsInCluster; + while (currReplicantsInTier < expectedReplicantsInTier) { + boolean replicate = currTotalReplicantsInCluster > 0; if (replicate && !replicationManager.canCreateReplicant(tier)) { break; @@ -116,7 +124,7 @@ public abstract class LoadRule implements Rule "Not enough [%s] servers or node capacity to assign segment[%s]! Expected Replicants[%d]", tier, segment.getIdentifier(), - expectedReplicants + expectedReplicantsInTier ); break; } @@ -143,8 +151,9 @@ public abstract class LoadRule implements Rule } ); - stats.addToTieredStat("assignedCount", tier, 1); - ++totalReplicants; + stats.addToTieredStat(assignedCount, tier, 1); + ++currReplicantsInTier; + ++currTotalReplicantsInCluster; } return stats; @@ -162,7 +171,7 @@ public abstract class LoadRule implements Rule return stats; } - // Make sure we have enough actual replicants in the correct tiers in the cluster before doing anything + // Make sure we have enough loaded replicants in the correct tiers in the cluster before doing anything for (Integer leftToLoad : loadStatus.values()) { if (leftToLoad > 0) { return stats; @@ -176,10 +185,10 @@ public abstract class LoadRule implements Rule for (Map.Entry entry : replicantsByTier.entrySet()) { final String tier = entry.getKey(); - int actualNumReplicantsForTier = entry.getValue(); + int loadedNumReplicantsForTier = entry.getValue(); int expectedNumReplicantsForTier = getNumReplicants(tier); - stats.addToTieredStat("droppedCount", tier, 0); + stats.addToTieredStat(droppedCount, tier, 0); MinMaxPriorityQueue serverQueue = params.getDruidCluster().get(tier); if (serverQueue == null) { @@ -188,7 +197,7 @@ public abstract class LoadRule implements Rule } List droppedServers = Lists.newArrayList(); - while (actualNumReplicantsForTier > expectedNumReplicantsForTier) { + while (loadedNumReplicantsForTier > expectedNumReplicantsForTier) { final ServerHolder holder = serverQueue.pollLast(); if (holder == null) { log.warn("Wtf, holder was null? I have no servers serving [%s]?", segment.getIdentifier()); @@ -224,8 +233,8 @@ public abstract class LoadRule implements Rule } } ); - --actualNumReplicantsForTier; - stats.addToTieredStat("droppedCount", tier, 1); + --loadedNumReplicantsForTier; + stats.addToTieredStat(droppedCount, tier, 1); } droppedServers.add(holder); } diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java index 2663a4a94e4..95d52c1fad1 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java @@ -979,6 +979,94 @@ public class DruidCoordinatorRuleRunnerTest EasyMock.verify(mockPeon); } + /** + * Nodes: + * hot - nothing loaded + * _default_tier - 1 segment loaded + * + * @throws Exception + */ + @Test + public void testReplicantThrottleAcrossTiers() throws Exception + { + mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject()); + EasyMock.expectLastCall().atLeastOnce(); + EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.newHashSet()).atLeastOnce(); + EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce(); + EasyMock.replay(mockPeon); + + EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn( + Lists.newArrayList( + new IntervalLoadRule( + new Interval("2012-01-01T00:00:00.000Z/2013-01-01T00:00:00.000Z"), + ImmutableMap.of( + "hot", 1, + DruidServer.DEFAULT_TIER, 1 + ), + null, + null + ) + ) + ).atLeastOnce(); + EasyMock.replay(databaseRuleManager); + + DruidCluster druidCluster = new DruidCluster( + ImmutableMap.of( + "hot", + MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create( + Arrays.asList( + new ServerHolder( + new DruidServer( + "serverHot", + "hostHot", + 1000, + "historical", + "hot", + 0 + ), + mockPeon + ) + ) + ), + DruidServer.DEFAULT_TIER, + MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create( + Arrays.asList( + new ServerHolder( + new DruidServer( + "serverNorm", + "hostNorm", + 1000, + "historical", + DruidServer.DEFAULT_TIER, + 0 + ), + mockPeon + ) + ) + ) + ) + ); + + DruidCoordinatorRuntimeParams params = + new DruidCoordinatorRuntimeParams.Builder() + .withDruidCluster(druidCluster) + .withAvailableSegments(availableSegments) + .withDatabaseRuleManager(databaseRuleManager) + .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) + .build(); + + DruidCoordinatorRuleRunner runner = new DruidCoordinatorRuleRunner(new ReplicationThrottler(7, 1), coordinator); + DruidCoordinatorRuntimeParams afterParams = runner.run(params); + CoordinatorStats stats = afterParams.getCoordinatorStats(); + + Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get("hot").get() == 24); + Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get(DruidServer.DEFAULT_TIER).get() == 7); + Assert.assertTrue(stats.getPerTierStats().get("unassignedCount") == null); + Assert.assertTrue(stats.getPerTierStats().get("unassignedSize") == null); + + EasyMock.verify(mockPeon); + } + @Test public void testDropReplicantThrottle() throws Exception { diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index a52927d0b7d..3e75d1b79c3 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -59,7 +59,7 @@ public class LoadRuleTest public void setUp() throws Exception { mockPeon = EasyMock.createMock(LoadQueuePeon.class); - throttler = new ReplicationThrottler(1, 1); + throttler = new ReplicationThrottler(2, 1); for (String tier : Arrays.asList("hot", DruidServer.DEFAULT_TIER)) { throttler.updateReplicationState(tier); throttler.updateTerminationState(tier); From 0ea45b590feb4f0c1c9089e1d0561e2cb6f9a09c Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:23:30 -0800 Subject: [PATCH 09/13] [maven-release-plugin] prepare release druid-0.6.56 --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index d35dcd94e91..c8f55e15a03 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/common/pom.xml b/common/pom.xml index 8877fb3f5f9..780138b3910 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/examples/pom.xml b/examples/pom.xml index d464bfd1348..4ec36caf36b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 203be72fa67..8965bc65a31 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/hll/pom.xml b/hll/pom.xml index c90622c740a..8b51cc4e0b3 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 18affe138a1..46514b00e93 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 5dbc2be0d9d..71110b5937d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 99fce7e8869..1101d19c292 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index a41bf9b0ab2..20bdc7e8ccd 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/pom.xml b/pom.xml index 51969af12c6..cb461a668e7 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.56-SNAPSHOT + 0.6.56 druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - ${project.artifactId}-${project.version} + druid-0.6.56 diff --git a/processing/pom.xml b/processing/pom.xml index a6add656a6b..761c92f0ac7 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index f255d6fdd00..49b2d1c4bed 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index d4eaabe9eac..b07cc632bc3 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/server/pom.xml b/server/pom.xml index 269bcd1b97a..412ee9bae53 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 diff --git a/services/pom.xml b/services/pom.xml index f443be47515..64712465d8c 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.56-SNAPSHOT + 0.6.56 From 056e82a51684a61b9faf07eee042007153b3f5db Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 4 Feb 2014 15:23:34 -0800 Subject: [PATCH 10/13] [maven-release-plugin] prepare for next development iteration --- cassandra-storage/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- hdfs-storage/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- kafka-eight/pom.xml | 2 +- kafka-seven/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- rabbitmq/pom.xml | 2 +- s3-extensions/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index c8f55e15a03..1172adf7564 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 780138b3910..497216bb8f6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 4ec36caf36b..1c4451a81e8 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index 8965bc65a31..7f8bdae4830 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index 8b51cc4e0b3..e566b93bb84 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 46514b00e93..6a960315b5f 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 71110b5937d..128c692d459 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index 1101d19c292..a049ab9cb47 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index 20bdc7e8ccd..d8af1ffe3a5 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/pom.xml b/pom.xml index cb461a668e7..eed8fa2bd0d 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,14 @@ io.druid druid pom - 0.6.56 + 0.6.57-SNAPSHOT druid druid scm:git:ssh://git@github.com/metamx/druid.git scm:git:ssh://git@github.com/metamx/druid.git http://www.github.com/metamx/druid - druid-0.6.56 + ${project.artifactId}-${project.version} diff --git a/processing/pom.xml b/processing/pom.xml index 761c92f0ac7..e9f51ce1228 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index 49b2d1c4bed..c407d4387d0 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index b07cc632bc3..6170c861d14 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 412ee9bae53..a6bf41d1f36 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 64712465d8c..1962aee650e 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.56 + 0.6.57-SNAPSHOT From 53c5124b992d40e67e779787097650ef42a0b6f7 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 5 Feb 2014 15:59:00 -0800 Subject: [PATCH 11/13] DbConnector: Indexes on tasks table --- common/src/main/java/io/druid/db/DbConnector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/io/druid/db/DbConnector.java b/common/src/main/java/io/druid/db/DbConnector.java index 03b8987297b..585276ade91 100644 --- a/common/src/main/java/io/druid/db/DbConnector.java +++ b/common/src/main/java/io/druid/db/DbConnector.java @@ -87,7 +87,8 @@ public class DbConnector + " `payload` longblob NOT NULL,\n" + " `status_payload` longblob NOT NULL,\n" + " `active` tinyint(1) NOT NULL DEFAULT '0',\n" - + " PRIMARY KEY (`id`)\n" + + " PRIMARY KEY (`id`),\n" + + " KEY (active, created_date(100))\n" + ")", taskTableName ) From af4827336939096ea179d60788d8e3dfa54af569 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 6 Feb 2014 11:19:05 -0800 Subject: [PATCH 12/13] fix bug with dynamic configs in coordinator not working --- .../overlord/http/OverlordResource.java | 1 - .../server/coordinator/DruidCoordinator.java | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java index 5a3eb1c5e07..58c917c0a26 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/http/OverlordResource.java @@ -46,7 +46,6 @@ import io.druid.indexing.overlord.scaling.ResourceManagementScheduler; import io.druid.indexing.overlord.setup.WorkerSetupData; import io.druid.tasklogs.TaskLogStreamer; import io.druid.timeline.DataSegment; -import org.apache.zookeeper.data.Stat; import org.joda.time.DateTime; import javax.ws.rs.Consumes; diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 78186119ee0..f5f202b5904 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -96,8 +96,8 @@ public class DruidCoordinator private final Object lock = new Object(); private volatile boolean started = false; + private volatile int leaderCounter = 0; private volatile boolean leader = false; - private volatile AtomicReference dynamicConfigs; private final DruidCoordinatorConfig config; private final ZkPathsConfig zkPaths; @@ -184,7 +184,6 @@ public class DruidCoordinator this.exec = scheduledExecutorFactory.create(1, "Coordinator-Exec--%d"); this.leaderLatch = new AtomicReference<>(null); - this.dynamicConfigs = new AtomicReference<>(null); this.loadManagementPeons = loadQueuePeonMap; } @@ -280,7 +279,11 @@ public class DruidCoordinator public CoordinatorDynamicConfig getDynamicConfigs() { - return dynamicConfigs.get(); + return configManager.watch( + CoordinatorDynamicConfig.CONFIG_KEY, + CoordinatorDynamicConfig.class, + new CoordinatorDynamicConfig.Builder().build() + ).get(); } public void removeSegment(DataSegment segment) @@ -533,6 +536,7 @@ public class DruidCoordinator log.info("I am the leader of the coordinators, all must bow!"); try { + leaderCounter++; leader = true; databaseSegmentManager.start(); databaseRuleManager.start(); @@ -540,11 +544,6 @@ public class DruidCoordinator serviceAnnouncer.announce(self); final List> coordinatorRunnables = Lists.newArrayList(); - dynamicConfigs = configManager.watch( - CoordinatorDynamicConfig.CONFIG_KEY, - CoordinatorDynamicConfig.class, - new CoordinatorDynamicConfig.Builder().build() - ); coordinatorRunnables.add(Pair.of(new CoordinatorHistoricalManagerRunnable(), config.getCoordinatorPeriod())); if (indexingServiceClient != null) { coordinatorRunnables.add( @@ -562,6 +561,7 @@ public class DruidCoordinator ); } + final int startingLeaderCounter = leaderCounter; for (final Pair coordinatorRunnable : coordinatorRunnables) { ScheduledExecutors.scheduleWithFixedDelay( exec, @@ -574,10 +574,10 @@ public class DruidCoordinator @Override public ScheduledExecutors.Signal call() { - if (leader) { + if (leader && startingLeaderCounter == leaderCounter) { theRunnable.run(); } - if (leader) { // (We might no longer be leader) + if (leader && startingLeaderCounter == leaderCounter) { // (We might no longer be leader) return ScheduledExecutors.Signal.REPEAT; } else { return ScheduledExecutors.Signal.STOP; @@ -610,6 +610,8 @@ public class DruidCoordinator { synchronized (lock) { try { + leaderCounter++; + log.info("I am no longer the leader..."); for (String server : loadManagementPeons.keySet()) { @@ -739,11 +741,10 @@ public class DruidCoordinator DruidCoordinatorRuntimeParams.newBuilder() .withStartTime(startTime) .withDatasources(databaseSegmentManager.getInventory()) - .withDynamicConfigs(dynamicConfigs.get()) + .withDynamicConfigs(getDynamicConfigs()) .withEmitter(emitter) .build(); - for (DruidCoordinatorHelper helper : helpers) { params = helper.run(params); } @@ -826,7 +827,6 @@ public class DruidCoordinator .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) - .withDynamicConfigs(dynamicConfigs.get()) .build(); } }, From 9756b18f9b5cf4f32438d42590d9d4e0fac47523 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 6 Feb 2014 11:26:05 -0800 Subject: [PATCH 13/13] sort complex metrics with groupby --- .../java/io/druid/query/groupby/orderby/DefaultLimitSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index 4cd052058bf..3bb79ed9617 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -148,7 +148,7 @@ public class DefaultLimitSpec implements LimitSpec @Override public int compare(Row left, Row right) { - return comparator.compare(left.getFloatMetric(column), right.getFloatMetric(column)); + return comparator.compare(left.getRaw(column), right.getRaw(column)); } }; }