From b82a4e57f3a2d160a67f7fca6523a6191b8a5f16 Mon Sep 17 00:00:00 2001 From: cheddar Date: Wed, 1 May 2013 17:31:29 -0500 Subject: [PATCH 01/11] 1) A "working" assembly --- druid-services/pom.xml | 18 +++++++++++- druid-services/src/assembly/assembly.xml | 36 ++++++++++++++++++++++++ druid-services/src/bin/run.sh | 1 + pom.xml | 5 ++++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 druid-services/src/assembly/assembly.xml create mode 100644 druid-services/src/bin/run.sh diff --git a/druid-services/pom.xml b/druid-services/pom.xml index 3928293ecec..2ac0e7ac9d7 100644 --- a/druid-services/pom.xml +++ b/druid-services/pom.xml @@ -42,7 +42,6 @@ druid-server ${project.parent.version} - @@ -64,6 +63,23 @@ + + maven-assembly-plugin + + + distro-assembly + package + + single + + + + src/assembly/assembly.xml + + + + + diff --git a/druid-services/src/assembly/assembly.xml b/druid-services/src/assembly/assembly.xml new file mode 100644 index 00000000000..c35642ba21d --- /dev/null +++ b/druid-services/src/assembly/assembly.xml @@ -0,0 +1,36 @@ + + bin + + dir + + false + + + src/bin + + *.sh + + 0644 + + + + + + + true + + + + com.metamx.druid:druid-server + com.metamx.druid:druid-indexer + com.metamx.druid:druid-realtime + + + modules/${artifactId} + false + + + + \ No newline at end of file diff --git a/druid-services/src/bin/run.sh b/druid-services/src/bin/run.sh new file mode 100644 index 00000000000..5543a6b0139 --- /dev/null +++ b/druid-services/src/bin/run.sh @@ -0,0 +1 @@ +echo "Hello World!" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 819c062aaef..40ac7fc22df 100644 --- a/pom.xml +++ b/pom.xml @@ -414,6 +414,11 @@ antlr4-maven-plugin 4.0 + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + From d1a8b93b3f83676592b5575769d3ada79629df97 Mon Sep 17 00:00:00 2001 From: cheddar Date: Wed, 1 May 2013 17:37:38 -0500 Subject: [PATCH 02/11] 1) druid-services -> services --- pom.xml | 2 +- {druid-services => services}/pom.xml | 0 {druid-services => services}/src/assembly/assembly.xml | 0 {druid-services => services}/src/bin/run.sh | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename {druid-services => services}/pom.xml (100%) rename {druid-services => services}/src/assembly/assembly.xml (100%) rename {druid-services => services}/src/bin/run.sh (100%) diff --git a/pom.xml b/pom.xml index 40ac7fc22df..14f77021fad 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ merger realtime examples - druid-services + services diff --git a/druid-services/pom.xml b/services/pom.xml similarity index 100% rename from druid-services/pom.xml rename to services/pom.xml diff --git a/druid-services/src/assembly/assembly.xml b/services/src/assembly/assembly.xml similarity index 100% rename from druid-services/src/assembly/assembly.xml rename to services/src/assembly/assembly.xml diff --git a/druid-services/src/bin/run.sh b/services/src/bin/run.sh similarity index 100% rename from druid-services/src/bin/run.sh rename to services/src/bin/run.sh From 12fe145cc87a145157f2e6851226d26ec5b955a7 Mon Sep 17 00:00:00 2001 From: cheddar Date: Thu, 2 May 2013 16:35:06 -0500 Subject: [PATCH 03/11] 1) Setup assembly to grab all dependencies -- it's broken right now, http://jira.codehaus.org/browse/MASSEMBLY-646 --- services/src/assembly/assembly.xml | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/services/src/assembly/assembly.xml b/services/src/assembly/assembly.xml index c35642ba21d..967ed9b2614 100644 --- a/services/src/assembly/assembly.xml +++ b/services/src/assembly/assembly.xml @@ -14,23 +14,18 @@ 0644 - - - - - - true - - + + / - com.metamx.druid:druid-server - com.metamx.druid:druid-indexer - com.metamx.druid:druid-realtime + LICENSE - - modules/${artifactId} - false - - - + + + + + true + true + lib/ + + \ No newline at end of file From 0b96585240fb4f0dad1aef9f29fcbacc5087bc93 Mon Sep 17 00:00:00 2001 From: cheddar Date: Thu, 2 May 2013 16:38:44 -0500 Subject: [PATCH 04/11] 1) Unbreak the build --- services/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/pom.xml b/services/pom.xml index 2ac0e7ac9d7..3926cbe986a 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -63,6 +63,7 @@ + From df6fd5eca7e4e1d4ee9c45ecd4e992beed74a4be Mon Sep 17 00:00:00 2001 From: cheddar Date: Thu, 2 May 2013 17:13:22 -0500 Subject: [PATCH 05/11] [maven-release-plugin] prepare release druid-0.4.1 --- client/pom.xml | 5 ++--- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 12 insertions(+), 13 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 36dcd62cac8..af9e71999ff 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -18,8 +18,7 @@ ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. --> - + 4.0.0 com.metamx.druid druid-client @@ -29,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/common/pom.xml b/common/pom.xml index 06bee0f0de0..034839b1425 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/examples/pom.xml b/examples/pom.xml index 1386b3ef956..95eaddbffed 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/index-common/pom.xml b/index-common/pom.xml index 0bae9b7a70d..be02503c457 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/indexer/pom.xml b/indexer/pom.xml index cbf998dbb48..7a5cf34cdb3 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/merger/pom.xml b/merger/pom.xml index 3f8a7e657b9..6135bf22fa9 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/pom.xml b/pom.xml index 14f77021fad..23eb3ffd095 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.1-SNAPSHOT + 0.4.1 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index c310f6ad0c5..2cb2999c74a 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/server/pom.xml b/server/pom.xml index 2f66678a1ea..5d2a3c68555 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 diff --git a/services/pom.xml b/services/pom.xml index 3926cbe986a..47771ae18bc 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.1-SNAPSHOT + 0.4.1 com.metamx druid - 0.4.1-SNAPSHOT + 0.4.1 From 5f184ea978d37fba9c4dd7b2087eca7802db8feb Mon Sep 17 00:00:00 2001 From: cheddar Date: Thu, 2 May 2013 17:13:28 -0500 Subject: [PATCH 06/11] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index af9e71999ff..ed3e2ecf6c2 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 034839b1425..3f3358be0b0 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 95eaddbffed..bd3d852f018 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index be02503c457..07a664410b0 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 7a5cf34cdb3..302ea3dc08f 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index 6135bf22fa9..cf14a48a062 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/pom.xml b/pom.xml index 23eb3ffd095..7a9b3f3a7dc 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.1 + 0.4.2-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 2cb2999c74a..f09fc2351d0 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 5d2a3c68555..09f71e10889 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 47771ae18bc..3acc6a04bfd 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.1 + 0.4.2-SNAPSHOT com.metamx druid - 0.4.1 + 0.4.2-SNAPSHOT From 3c04f1635c95083b8936777af468238075efec87 Mon Sep 17 00:00:00 2001 From: cheddar Date: Thu, 2 May 2013 18:00:50 -0500 Subject: [PATCH 07/11] 1) Fix bug with same name on multiple JSON properties --- common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java index ce1c14c08b0..fb7d1a99916 100644 --- a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java +++ b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java @@ -42,7 +42,7 @@ public abstract class DbConnectorConfig @Config("druid.database.segmentTable") public abstract String getSegmentTable(); - @JsonProperty("validationQuery") + @JsonProperty("useValidationQuery") @Config("druid.database.validation") public boolean isValidationQuery() { return false; From 6649a4b38d90455c1669939898c455ae19bc41dd Mon Sep 17 00:00:00 2001 From: cheddar Date: Fri, 3 May 2013 12:13:57 -0500 Subject: [PATCH 08/11] 1) Don't require druid.zk.paths.base when attempting to load properties from zk --- .../druid/initialization/Initialization.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/initialization/Initialization.java b/client/src/main/java/com/metamx/druid/initialization/Initialization.java index 36c89c3aba1..73166c54eca 100644 --- a/client/src/main/java/com/metamx/druid/initialization/Initialization.java +++ b/client/src/main/java/com/metamx/druid/initialization/Initialization.java @@ -121,29 +121,39 @@ public class Initialization if (tmp_props.getProperty(zkHostsProperty) != null) { final ConfigurationObjectFactory factory = Config.createFactory(tmp_props); - Lifecycle lifecycle = new Lifecycle(); + ZkPathsConfig config; try { - final ZkPathsConfig config = factory.build(ZkPathsConfig.class); - CuratorFramework curator = makeCuratorFramework(factory.build(CuratorConfig.class), lifecycle); - - lifecycle.start(); - - final Stat stat = curator.checkExists().forPath(config.getPropertiesPath()); - if (stat != null) { - final byte[] data = curator.getData().forPath(config.getPropertiesPath()); - zkProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - } - - // log properties from zk - for (String prop : zkProps.stringPropertyNames()) { - log.info("Loaded(zk) Property[%s] as [%s]", prop, zkProps.getProperty(prop)); - } + config = factory.build(ZkPathsConfig.class); } - catch (Exception e) { - throw Throwables.propagate(e); + catch (IllegalArgumentException e) { + log.warn(e, "Unable to build ZkPathsConfig. Cannot load properties from ZK."); + config = null; } - finally { - lifecycle.stop(); + + if (config != null) { + Lifecycle lifecycle = new Lifecycle(); + try { + CuratorFramework curator = makeCuratorFramework(factory.build(CuratorConfig.class), lifecycle); + + lifecycle.start(); + + final Stat stat = curator.checkExists().forPath(config.getPropertiesPath()); + if (stat != null) { + final byte[] data = curator.getData().forPath(config.getPropertiesPath()); + zkProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); + } + + // log properties from zk + for (String prop : zkProps.stringPropertyNames()) { + log.info("Loaded(zk) Property[%s] as [%s]", prop, zkProps.getProperty(prop)); + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + finally { + lifecycle.stop(); + } } } else { log.warn("property[%s] not set, skipping ZK-specified properties.", zkHostsProperty); From 4fef7c2c4313247de010a1ffab48686e9a6b79c7 Mon Sep 17 00:00:00 2001 From: cheddar Date: Fri, 3 May 2013 12:16:57 -0500 Subject: [PATCH 09/11] [maven-release-plugin] prepare release druid-0.4.2 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index ed3e2ecf6c2..d6d8254a137 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/common/pom.xml b/common/pom.xml index 3f3358be0b0..e25f61fb0fc 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/examples/pom.xml b/examples/pom.xml index bd3d852f018..4b96909d389 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/index-common/pom.xml b/index-common/pom.xml index 07a664410b0..b75fed82bf6 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/indexer/pom.xml b/indexer/pom.xml index 302ea3dc08f..95e9464abb0 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/merger/pom.xml b/merger/pom.xml index cf14a48a062..f9e13b0d658 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/pom.xml b/pom.xml index 7a9b3f3a7dc..f750178d718 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.2-SNAPSHOT + 0.4.2 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index f09fc2351d0..6ebc329b17a 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/server/pom.xml b/server/pom.xml index 09f71e10889..c0e2529b32e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 diff --git a/services/pom.xml b/services/pom.xml index 3acc6a04bfd..4743d0e2492 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.2-SNAPSHOT + 0.4.2 com.metamx druid - 0.4.2-SNAPSHOT + 0.4.2 From c89f4cc85f4fe0993667840706703f798d68f0b3 Mon Sep 17 00:00:00 2001 From: cheddar Date: Fri, 3 May 2013 12:17:05 -0500 Subject: [PATCH 10/11] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index d6d8254a137..6756f8cd9a7 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index e25f61fb0fc..3a3a7e1c940 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 4b96909d389..f95daba8096 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index b75fed82bf6..cfe1b68508b 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 95e9464abb0..37ce4935b09 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index f9e13b0d658..a316d2729e5 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index f750178d718..7fc9fbe7438 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.2 + 0.4.3-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 6ebc329b17a..4d5c921b40d 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index c0e2529b32e..cc15a8331da 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 4743d0e2492..eccf9b093c0 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.2 + 0.4.3-SNAPSHOT com.metamx druid - 0.4.2 + 0.4.3-SNAPSHOT From c15ac1fc908e8a5db84cafc19cae84ef6c9ead66 Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 3 May 2013 10:52:00 -0700 Subject: [PATCH 11/11] limit spec for group by --- .../druid/query/group/DefaultLimitSpec.java | 150 ++++++++++++++++++ .../druid/query/group/GroupByQuery.java | 20 +++ .../group/GroupByQueryQueryToolChest.java | 49 ++++-- .../metamx/druid/query/group/LimitSpec.java | 44 +++++ pom.xml | 2 +- 5 files changed, 253 insertions(+), 12 deletions(-) create mode 100644 client/src/main/java/com/metamx/druid/query/group/DefaultLimitSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/LimitSpec.java diff --git a/client/src/main/java/com/metamx/druid/query/group/DefaultLimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/DefaultLimitSpec.java new file mode 100644 index 00000000000..4a09a6dfb79 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/DefaultLimitSpec.java @@ -0,0 +1,150 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package com.metamx.druid.query.group; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.metamx.common.ISE; +import com.metamx.druid.input.MapBasedRow; +import com.metamx.druid.input.Row; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; + +/** + */ +public class DefaultLimitSpec implements LimitSpec +{ + private static final byte CACHE_TYPE_ID = 0x0; + private static Joiner JOINER = Joiner.on(""); + + private final List orderBy; + private final int limit; + private final Comparator comparator; + + @JsonCreator + public DefaultLimitSpec( + @JsonProperty("orderBy") List orderBy, + @JsonProperty("limit") int limit + ) + { + this.orderBy = (orderBy == null) ? Lists.newArrayList() : orderBy; + this.limit = limit; + this.comparator = makeComparator(); + } + + public DefaultLimitSpec() + { + this.orderBy = Lists.newArrayList(); + this.limit = 0; + this.comparator = makeComparator(); + } + + @JsonProperty + @Override + public List getOrderBy() + { + return orderBy; + } + + @JsonProperty + @Override + public int getLimit() + { + return limit; + } + + @Override + public Comparator getComparator() + { + return comparator; + } + + @Override + public byte[] getCacheKey() + { + byte[] orderByBytes = JOINER.join(orderBy).getBytes(); + + byte[] limitBytes = Ints.toByteArray(limit); + + return ByteBuffer.allocate(1 + orderByBytes.length + limitBytes.length) + .put(CACHE_TYPE_ID) + .put(orderByBytes) + .put(limitBytes) + .array(); + } + + @Override + public String toString() + { + return "DefaultLimitSpec{" + + "orderBy='" + orderBy + '\'' + + ", limit=" + limit + + '}'; + } + + private Comparator makeComparator() + { + Ordering ordering = new Ordering() + { + @Override + public int compare(Row left, Row right) + { + return Longs.compare(left.getTimestampFromEpoch(), right.getTimestampFromEpoch()); + } + }; + + for (final String dimension : orderBy) { + ordering = ordering.compound( + new Comparator() + { + @Override + public int compare(Row left, Row right) + { + if (left instanceof MapBasedRow && right instanceof MapBasedRow) { + // There are no multi-value dimensions at this point, they should have been flattened out + String leftDimVal = left.getDimension(dimension).get(0); + String rightDimVal = right.getDimension(dimension).get(0); + return leftDimVal.compareTo(rightDimVal); + } else { + throw new ISE("Unknown type for rows[%s, %s]", left.getClass(), right.getClass()); + } + } + } + ); + } + final Ordering theOrdering = ordering; + + return new Comparator() + { + @Override + public int compare(Row row, Row row2) + { + return theOrdering.compare(row, row2); + } + }; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java b/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java index df8ccedaa7a..6f8eb052cb1 100644 --- a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java +++ b/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java @@ -49,6 +49,7 @@ public class GroupByQuery extends BaseQuery return new Builder(); } + private final LimitSpec limitSpec; private final DimFilter dimFilter; private final QueryGranularity granularity; private final List dimensions; @@ -59,6 +60,7 @@ public class GroupByQuery extends BaseQuery public GroupByQuery( @JsonProperty("dataSource") String dataSource, @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, + @JsonProperty("limitSpec") LimitSpec limitSpec, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @@ -68,6 +70,7 @@ public class GroupByQuery extends BaseQuery ) { super(dataSource, querySegmentSpec, context); + this.limitSpec = (limitSpec == null) ? new DefaultLimitSpec() : limitSpec; this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions == null ? ImmutableList.of() : dimensions; @@ -79,6 +82,12 @@ public class GroupByQuery extends BaseQuery Queries.verifyAggregations(this.aggregatorSpecs, this.postAggregatorSpecs); } + @JsonProperty + public LimitSpec getLimitSpec() + { + return limitSpec; + } + @JsonProperty("filter") public DimFilter getDimFilter() { @@ -127,6 +136,7 @@ public class GroupByQuery extends BaseQuery return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + limitSpec, dimFilter, granularity, dimensions, @@ -142,6 +152,7 @@ public class GroupByQuery extends BaseQuery return new GroupByQuery( getDataSource(), spec, + limitSpec, dimFilter, granularity, dimensions, @@ -155,6 +166,7 @@ public class GroupByQuery extends BaseQuery { private String dataSource; private QuerySegmentSpec querySegmentSpec; + private LimitSpec limitSpec; private DimFilter dimFilter; private QueryGranularity granularity; private List dimensions; @@ -168,6 +180,7 @@ public class GroupByQuery extends BaseQuery { dataSource = builder.dataSource; querySegmentSpec = builder.querySegmentSpec; + limitSpec = builder.limitSpec; dimFilter = builder.dimFilter; granularity = builder.granularity; dimensions = builder.dimensions; @@ -187,6 +200,12 @@ public class GroupByQuery extends BaseQuery return setQuerySegmentSpec(new LegacySegmentSpec(interval)); } + public Builder setLimitSpec(LimitSpec limitSpec) + { + this.limitSpec = limitSpec; + return this; + } + public Builder setQuerySegmentSpec(QuerySegmentSpec querySegmentSpec) { this.querySegmentSpec = querySegmentSpec; @@ -279,6 +298,7 @@ public class GroupByQuery extends BaseQuery return new GroupByQuery( dataSource, querySegmentSpec, + limitSpec, dimFilter, granularity, dimensions, diff --git a/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java b/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java index f6e63b36acb..f7e81fea4d0 100644 --- a/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java +++ b/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java @@ -21,10 +21,12 @@ package com.metamx.druid.query.group; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; -import com.google.common.base.Functions; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Longs; import com.metamx.common.ISE; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.ConcatSequence; @@ -44,11 +46,11 @@ import com.metamx.druid.query.QueryToolChest; import com.metamx.druid.query.dimension.DimensionSpec; import com.metamx.druid.utils.PropUtils; import com.metamx.emitter.service.ServiceMetricEvent; - import org.joda.time.Interval; import org.joda.time.Minutes; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -57,8 +59,9 @@ import java.util.Properties; */ public class GroupByQueryQueryToolChest extends QueryToolChest { - - private static final TypeReference TYPE_REFERENCE = new TypeReference(){}; + private static final TypeReference TYPE_REFERENCE = new TypeReference() + { + }; private static final String GROUP_BY_MERGE_KEY = "groupByMerge"; private static final Map NO_MERGE_CONTEXT = ImmutableMap.of(GROUP_BY_MERGE_KEY, "false"); @@ -81,22 +84,21 @@ public class GroupByQueryQueryToolChest extends QueryToolChest mergeGroupByResults(GroupByQuery query, QueryRunner runner) + private Sequence mergeGroupByResults(final GroupByQuery query, QueryRunner runner) { final QueryGranularity gran = query.getGranularity(); final long timeStart = query.getIntervals().get(0).getStartMillis(); // use gran.iterable instead of gran.truncate so that // AllGranularity returns timeStart instead of Long.MIN_VALUE - final long granTimeStart = gran.iterable(timeStart, timeStart+1).iterator().next(); + final long granTimeStart = gran.iterable(timeStart, timeStart + 1).iterator().next(); final List aggs = Lists.transform( query.getAggregatorSpecs(), @@ -144,7 +146,7 @@ public class GroupByQueryQueryToolChest extends QueryToolChest retVal = Sequences.map( Sequences.simple(index.iterableWithPostAggregations(query.getPostAggregatorSpecs())), new Function() { @@ -156,6 +158,16 @@ public class GroupByQueryQueryToolChest extends QueryToolChest 0) ? query.getLimitSpec().getLimit() : maxRows + ); } @Override @@ -183,9 +195,24 @@ public class GroupByQueryQueryToolChest extends QueryToolChest makeMetricManipulatorFn(GroupByQuery query, MetricManipulationFn fn) + public Function makeMetricManipulatorFn(final GroupByQuery query, final MetricManipulationFn fn) { - return Functions.identity(); + return new Function() + { + @Override + public Row apply(Row input) + { + if (input instanceof MapBasedRow) { + final MapBasedRow inputRow = (MapBasedRow) input; + final Map values = Maps.newHashMap(((MapBasedRow) input).getEvent()); + for (AggregatorFactory agg : query.getAggregatorSpecs()) { + values.put(agg.getName(), fn.manipulate(agg, inputRow.getEvent().get(agg.getName()))); + } + return new MapBasedRow(inputRow.getTimestamp(), values); + } + return input; + } + }; } @Override diff --git a/client/src/main/java/com/metamx/druid/query/group/LimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/LimitSpec.java new file mode 100644 index 00000000000..dab5414a16a --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/LimitSpec.java @@ -0,0 +1,44 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package com.metamx.druid.query.group; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.metamx.druid.input.Row; + +import java.util.Comparator; +import java.util.List; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "default", value = DefaultLimitSpec.class) +}) +public interface LimitSpec +{ + public List getOrderBy(); + + public int getLimit(); + + public Comparator getComparator(); + + public byte[] getCacheKey(); +} diff --git a/pom.xml b/pom.xml index 7fc9fbe7438..e79dd0aaa57 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ UTF-8 - 0.22.0 + 0.22.3 2.0.1-21-22