From bec5d248f04c83a0f00a26e582580c7500392191 Mon Sep 17 00:00:00 2001 From: xvrl Date: Wed, 8 May 2013 15:38:45 -0700 Subject: [PATCH 01/28] fix typo --- .../main/java/com/metamx/druid/client/ServerInventoryView.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java b/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java index b15b547a5f2..4c6b302ad46 100644 --- a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java +++ b/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java @@ -135,7 +135,7 @@ public class ServerInventoryView implements ServerView, InventoryView @Override public void deadContainer(DruidServer deadContainer) { - log.info("Server Disdappeared[%s]", deadContainer); + log.info("Server Disappeared[%s]", deadContainer); runServerCallbacks(deadContainer); } From 3e3f2c2da7bcc7ae06cee1ac738f99654516c8cd Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 9 May 2013 15:02:10 -0700 Subject: [PATCH 02/28] HadoopDruidIndexerJob: Correct error message when jobs fail --- .../java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java index 8dad62cd2c6..f2f192fbeea 100644 --- a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java +++ b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java @@ -102,8 +102,6 @@ public class HadoopDruidIndexerJob implements Jobby } } - publishedSegments = IndexGeneratorJob.getPublishedSegments(config); - if (!config.isLeaveIntermediate()) { if (failedMessage == null || config.isCleanupOnFailure()) { Path workingPath = config.makeIntermediatePath(); @@ -121,6 +119,8 @@ public class HadoopDruidIndexerJob implements Jobby throw new ISE(failedMessage); } + publishedSegments = IndexGeneratorJob.getPublishedSegments(config); + return true; } From 6917ff02d87a5a8cb522f892a48382a73448943d Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 9 May 2013 16:53:29 -0700 Subject: [PATCH 03/28] first commit --- .../metamx/druid/client/BrokerServerView.java | 27 +++-- .../druid/client/CachingClusteredClient.java | 2 +- .../druid/client/DirectDruidClient.java | 14 ++- .../client/selector/QueryableDruidServer.java | 28 +++++ .../druid/client/selector/ServerSelector.java | 32 +++--- .../client/selector/ServerSelectorTest.java | 102 ++++++++++++++++++ 6 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java create mode 100644 client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java diff --git a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java b/client/src/main/java/com/metamx/druid/client/BrokerServerView.java index 54a67257692..ccb28464c6a 100644 --- a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java +++ b/client/src/main/java/com/metamx/druid/client/BrokerServerView.java @@ -24,6 +24,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.metamx.common.logger.Logger; import com.metamx.druid.VersionedIntervalTimeline; +import com.metamx.druid.client.selector.QueryableDruidServer; import com.metamx.druid.client.selector.ServerSelector; import com.metamx.druid.partition.PartitionChunk; import com.metamx.druid.query.QueryRunner; @@ -44,7 +45,7 @@ public class BrokerServerView implements TimelineServerView private final Object lock = new Object(); - private final ConcurrentMap clients; + private final ConcurrentMap clients; private final Map selectors; private final Map> timelines; @@ -107,7 +108,7 @@ public class BrokerServerView implements TimelineServerView public void clear() { synchronized (lock) { - final Iterator clientsIter = clients.keySet().iterator(); + final Iterator clientsIter = clients.keySet().iterator(); while (clientsIter.hasNext()) { clientsIter.remove(); } @@ -119,7 +120,7 @@ public class BrokerServerView implements TimelineServerView final ServerSelector selector = selectorsIter.next(); selectorsIter.remove(); while (!selector.isEmpty()) { - final DruidServer pick = selector.pick(); + final QueryableDruidServer pick = selector.pick(); selector.removeServer(pick); } } @@ -128,7 +129,10 @@ public class BrokerServerView implements TimelineServerView private void addServer(DruidServer server) { - QueryRunner exists = clients.put(server, makeDirectClient(server)); + QueryableDruidServer exists = clients.put( + server.getName(), + new QueryableDruidServer(server, makeDirectClient(server)) + ); if (exists != null) { log.warn("QueryRunner for server[%s] already existed!?", server); } @@ -141,7 +145,7 @@ public class BrokerServerView implements TimelineServerView private void removeServer(DruidServer server) { - clients.remove(server); + clients.remove(server.getName()); for (DataSegment segment : server.getSegments().values()) { serverRemovedSegment(server, segment); } @@ -167,10 +171,10 @@ public class BrokerServerView implements TimelineServerView selectors.put(segmentId, selector); } - if (!clients.containsKey(server)) { + if (!clients.containsKey(server.getName())) { addServer(server); } - selector.addServer(server); + selector.addServer(clients.get(server.getName())); } } @@ -188,7 +192,8 @@ public class BrokerServerView implements TimelineServerView return; } - if (!selector.removeServer(server)) { + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (!selector.removeServer(queryableDruidServer)) { log.warn( "Asked to disassociate non-existant association between server[%s] and segment[%s]", server, @@ -228,7 +233,11 @@ public class BrokerServerView implements TimelineServerView public QueryRunner getQueryRunner(DruidServer server) { synchronized (lock) { - return clients.get(server); + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.error("WTF?! No QueryableDruidServer found for %s", server.getName()); + } + return queryableDruidServer.getClient(); } } diff --git a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java b/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java index 261ed51e136..eba9b399ccd 100644 --- a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java +++ b/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java @@ -203,7 +203,7 @@ public class CachingClusteredClient implements QueryRunner // Compile list of all segments not pulled from cache for(Pair segment : segments) { - final DruidServer server = segment.lhs.pick(); + final DruidServer server = segment.lhs.pick().getServer(); List descriptors = serverSegments.get(server); if (descriptors == null) { diff --git a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java b/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java index 755c79d7cc3..445ea44dec1 100644 --- a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java +++ b/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java @@ -60,6 +60,8 @@ import java.util.Iterator; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** */ @@ -74,6 +76,7 @@ public class DirectDruidClient implements QueryRunner private final HttpClient httpClient; private final String host; + private final AtomicInteger openConnections; private final boolean isSmile; public DirectDruidClient( @@ -88,7 +91,13 @@ public class DirectDruidClient implements QueryRunner this.httpClient = httpClient; this.host = host; - isSmile = this.objectMapper.getJsonFactory() instanceof SmileFactory; + this.isSmile = this.objectMapper.getJsonFactory() instanceof SmileFactory; + this.openConnections = new AtomicInteger(); + } + + public int getNumOpenConnections() + { + return openConnections.get(); } @Override @@ -121,6 +130,7 @@ public class DirectDruidClient implements QueryRunner try { log.debug("Querying url[%s]", url); + openConnections.getAndIncrement(); future = httpClient .post(new URL(url)) .setContent(objectMapper.writeValueAsBytes(query)) @@ -128,7 +138,6 @@ public class DirectDruidClient implements QueryRunner .go( new InputStreamResponseHandler() { - long startTime; long byteCount = 0; @@ -162,6 +171,7 @@ public class DirectDruidClient implements QueryRunner stopTime - startTime, byteCount / (0.0001 * (stopTime - startTime)) ); + openConnections.getAndDecrement(); return super.done(clientResponse); } } diff --git a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java b/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java new file mode 100644 index 00000000000..69a738bc157 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java @@ -0,0 +1,28 @@ +package com.metamx.druid.client.selector; + +import com.metamx.druid.client.DirectDruidClient; +import com.metamx.druid.client.DruidServer; + +/** + */ +public class QueryableDruidServer +{ + private final DruidServer server; + private final DirectDruidClient client; + + public QueryableDruidServer(DruidServer server, DirectDruidClient client) + { + this.server = server; + this.client = client; + } + + public DruidServer getServer() + { + return server; + } + + public DirectDruidClient getClient() + { + return client; + } +} diff --git a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java b/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java index 22d56e46f69..086b117b59a 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java +++ b/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java @@ -19,21 +19,30 @@ package com.metamx.druid.client.selector; -import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; import com.metamx.druid.client.DataSegment; import com.metamx.druid.client.DruidServer; -import java.util.LinkedHashSet; -import java.util.Random; +import java.util.Collections; +import java.util.Comparator; +import java.util.Set; /** */ public class ServerSelector { - private static final Random random = new Random(); + private static final Comparator comparator = new Comparator() + { + @Override + public int compare(QueryableDruidServer left, QueryableDruidServer right) + { + return Ints.compare(left.getClient().getNumOpenConnections(), right.getClient().getNumOpenConnections()); + } + }; + + private final Set servers = Sets.newHashSet(); - private final LinkedHashSet servers = Sets.newLinkedHashSet(); private final DataSegment segment; public ServerSelector( @@ -49,7 +58,7 @@ public class ServerSelector } public void addServer( - DruidServer server + QueryableDruidServer server ) { synchronized (this) { @@ -57,7 +66,7 @@ public class ServerSelector } } - public boolean removeServer(DruidServer server) + public boolean removeServer(QueryableDruidServer server) { synchronized (this) { return servers.remove(server); @@ -71,15 +80,10 @@ public class ServerSelector } } - public DruidServer pick() + public QueryableDruidServer pick() { synchronized (this) { - final int size = servers.size(); - switch (size) { - case 0: return null; - case 1: return servers.iterator().next(); - default: return Iterables.get(servers, random.nextInt(size)); - } + return Collections.min(servers, comparator); } } } diff --git a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java b/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java new file mode 100644 index 00000000000..cf143b7480f --- /dev/null +++ b/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java @@ -0,0 +1,102 @@ +package com.metamx.druid.client.selector; + +import com.fasterxml.jackson.dataformat.smile.SmileFactory; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.SettableFuture; +import com.metamx.druid.Druids; +import com.metamx.druid.client.DataSegment; +import com.metamx.druid.client.DirectDruidClient; +import com.metamx.druid.jackson.DefaultObjectMapper; +import com.metamx.druid.query.ReflectionQueryToolChestWarehouse; +import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; +import com.metamx.druid.shard.NoneShardSpec; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.Request; +import com.metamx.http.client.RequestBuilder; +import junit.framework.Assert; +import org.easymock.EasyMock; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + +/** + */ +public class ServerSelectorTest +{ + private HttpClient httpClient; + + @Before + public void setUp() throws Exception + { + httpClient = EasyMock.createMock(HttpClient.class); + } + + @Test + public void testPick() throws Exception + { + RequestBuilder requestBuilder = new RequestBuilder(httpClient, HttpMethod.POST, new URL("http://foo.com")); + EasyMock.expect(httpClient.post(EasyMock.anyObject())).andReturn(requestBuilder).atLeastOnce(); + EasyMock.expect(httpClient.go(EasyMock.anyObject())).andReturn(SettableFuture.create()).atLeastOnce(); + EasyMock.replay(httpClient); + + final ServerSelector serverSelector = new ServerSelector( + new DataSegment( + "test", + new Interval("2013-01-01/2013-01-02"), + new DateTime("2013-01-01").toString(), + Maps.newHashMap(), + Lists.newArrayList(), + Lists.newArrayList(), + new NoneShardSpec(), + 0, + 0L + ) + ); + + DirectDruidClient client1 = new DirectDruidClient( + new ReflectionQueryToolChestWarehouse(), + new DefaultObjectMapper(new SmileFactory()), + httpClient, + "foo" + ); + DirectDruidClient client2 = new DirectDruidClient( + new ReflectionQueryToolChestWarehouse(), + new DefaultObjectMapper(new SmileFactory()), + httpClient, + "foo2" + ); + + QueryableDruidServer queryableDruidServer1 = new QueryableDruidServer( + null, + client1 + ); + serverSelector.addServer(queryableDruidServer1); + QueryableDruidServer queryableDruidServer2 = new QueryableDruidServer( + null, + client2 + ); + serverSelector.addServer(queryableDruidServer2); + + TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource("test").build(); + + client1.run(query); + client1.run(query); + client1.run(query); + + Assert.assertTrue(client1.getNumOpenConnections() == 3); + + client2.run(query); + client2.run(query); + + Assert.assertTrue(client2.getNumOpenConnections() == 2); + + Assert.assertTrue(serverSelector.pick() == queryableDruidServer2); + + EasyMock.verify(httpClient); + } +} From 7cc30a0747284828e240f9a96a7aecc84a4e6805 Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 10 May 2013 11:04:58 -0700 Subject: [PATCH 04/28] cleanup --- .../client/selector/QueryableDruidServer.java | 19 +++++++++++++++++++ .../client/selector/ServerSelectorTest.java | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java b/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java index 69a738bc157..2528facb8d8 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java +++ b/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package com.metamx.druid.client.selector; import com.metamx.druid.client.DirectDruidClient; diff --git a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java b/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java index cf143b7480f..9009ec3d246 100644 --- a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java +++ b/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package com.metamx.druid.client.selector; import com.fasterxml.jackson.dataformat.smile.SmileFactory; From bcfeac2d8c4d75589c1c74d52bb9c92cb1596223 Mon Sep 17 00:00:00 2001 From: fjy Date: Sun, 12 May 2013 15:14:57 -0700 Subject: [PATCH 05/28] fix NPE --- .../com/metamx/druid/coordination/ServerManager.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java index 155d9e20793..00721d042ad 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java +++ b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java @@ -20,6 +20,7 @@ package com.metamx.druid.coordination; import com.google.common.base.Function; +import com.google.common.base.Predicate; import com.google.common.collect.Ordering; import com.metamx.common.ISE; import com.metamx.common.guava.FunctionalIterable; @@ -257,6 +258,16 @@ public class ServerManager implements QuerySegmentWalker ); } } + ) + .filter( + new Predicate>() + { + @Override + public boolean apply(@Nullable QueryRunner input) + { + return (input != null); + } + } ); return new FinalizeResultsQueryRunner(toolChest.mergeResults(factory.mergeRunners(exec, adapters)), toolChest); From 89eb2aff5a0094a018c4535d17caaaa1fb77c24e Mon Sep 17 00:00:00 2001 From: fjy Date: Sun, 12 May 2013 15:17:09 -0700 Subject: [PATCH 06/28] fix logging --- .../com/metamx/druid/merger/coordinator/RemoteTaskRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java index f8d71836624..2542d825422 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java @@ -687,7 +687,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider ); if (workerQueue.isEmpty()) { - log.info("Worker nodes %s do not have capacity to run any more tasks!", zkWorkers.values()); + log.debug("Worker nodes %s do not have capacity to run any more tasks!", zkWorkers.values()); return null; } From 64febd47d76ead8350885aaa45317e0c4ba67ae1 Mon Sep 17 00:00:00 2001 From: fjy Date: Sun, 12 May 2013 15:19:12 -0700 Subject: [PATCH 07/28] [maven-release-plugin] prepare release druid-0.4.9 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index be24bb81ad4..ab402f80432 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/common/pom.xml b/common/pom.xml index b8757761ce2..6186e6a9ebe 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/examples/pom.xml b/examples/pom.xml index 8d82d054d1a..63593c11b0c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/index-common/pom.xml b/index-common/pom.xml index 4fccfbeaf0a..31bf2f10437 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/indexer/pom.xml b/indexer/pom.xml index dc2729cc48f..e264be90147 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/merger/pom.xml b/merger/pom.xml index 43e33f7e82b..e4d736b84c8 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/pom.xml b/pom.xml index 195e87473e0..f6744b5be8a 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.9-SNAPSHOT + 0.4.9 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index de3cf71ae08..49b605e7181 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/server/pom.xml b/server/pom.xml index ad8b5253a19..02697d81082 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 diff --git a/services/pom.xml b/services/pom.xml index b4cf8354c54..24ed287cc71 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.9-SNAPSHOT + 0.4.9 com.metamx druid - 0.4.9-SNAPSHOT + 0.4.9 From 219a8d7d05256cdeb045083c694871603533cf2a Mon Sep 17 00:00:00 2001 From: fjy Date: Sun, 12 May 2013 15:19:19 -0700 Subject: [PATCH 08/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index ab402f80432..488534c7e9b 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 6186e6a9ebe..08a3134addf 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 63593c11b0c..96b24ad1c8c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 31bf2f10437..c27f6b7c1ca 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index e264be90147..2dc8a7e9f10 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index e4d736b84c8..c359719ddc6 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/pom.xml b/pom.xml index f6744b5be8a..98f73fd5a05 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.9 + 0.4.10-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 49b605e7181..7877aac9f71 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 02697d81082..514059b3fc6 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 24ed287cc71..1a7126c6210 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.9 + 0.4.10-SNAPSHOT com.metamx druid - 0.4.9 + 0.4.10-SNAPSHOT From 1a978fe5252d1b2b61f045da7c06d847ad4ca41a Mon Sep 17 00:00:00 2001 From: fjy Date: Sun, 12 May 2013 15:26:19 -0700 Subject: [PATCH 09/28] fix predicates --- .../com/metamx/druid/coordination/ServerManager.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java index 00721d042ad..dceb3b9fafa 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java +++ b/server/src/main/java/com/metamx/druid/coordination/ServerManager.java @@ -20,7 +20,7 @@ package com.metamx.druid.coordination; import com.google.common.base.Function; -import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.collect.Ordering; import com.metamx.common.ISE; import com.metamx.common.guava.FunctionalIterable; @@ -260,14 +260,7 @@ public class ServerManager implements QuerySegmentWalker } ) .filter( - new Predicate>() - { - @Override - public boolean apply(@Nullable QueryRunner input) - { - return (input != null); - } - } + Predicates.>notNull() ); return new FinalizeResultsQueryRunner(toolChest.mergeResults(factory.mergeRunners(exec, adapters)), toolChest); From 9ab22f238e32d0a1bbd0613749a8bf6c5e08c3ca Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 11:06:46 -0700 Subject: [PATCH 10/28] fix bug in RTR for not correctly exiting after timeout --- .../merger/coordinator/RemoteTaskRunner.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java index 2542d825422..fe0577a8c77 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java @@ -230,6 +230,13 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider return null; } + public boolean isWorkerRunningTask(String workerHost, String taskId) + { + ZkWorker zkWorker = zkWorkers.get(workerHost); + + return (zkWorker != null && zkWorker.getRunningTasks().contains(taskId)); + } + /** * A task will be run only if there is no current knowledge in the RemoteTaskRunner of the task. * @@ -350,6 +357,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider /** * Adds a task to the pending queue + * * @param taskRunnerWorkItem */ private void addPendingTask(final TaskRunnerWorkItem taskRunnerWorkItem) @@ -506,8 +514,18 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider // Syncing state with Zookeeper - don't assign new tasks until the task we just assigned is actually running // on a worker - this avoids overflowing a worker with tasks synchronized (statusLock) { - while (findWorkerRunningTask(task.getId()) == null) { + while (!isWorkerRunningTask(theWorker.getHost(), task.getId())) { statusLock.wait(config.getTaskAssignmentTimeoutDuration().getMillis()); + if (!isWorkerRunningTask(theWorker.getHost(), task.getId())) { + log.error( + "Something went wrong! %s never ran task %s after %s!", + theWorker.getHost(), + task.getId(), + config.getTaskAssignmentTimeoutDuration() + ); + retryTask(runningTasks.get(task.getId()), theWorker.getHost()); + break; + } } } } From 1860104abe686b0edf1ba5b2cb33fe7a63c28ffe Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 11:08:29 -0700 Subject: [PATCH 11/28] [maven-release-plugin] prepare release druid-0.4.10 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 488534c7e9b..6bd0b1693c7 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/common/pom.xml b/common/pom.xml index 08a3134addf..6a9ddf1c915 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/examples/pom.xml b/examples/pom.xml index 96b24ad1c8c..34e38e481a5 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/index-common/pom.xml b/index-common/pom.xml index c27f6b7c1ca..f67f941af27 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/indexer/pom.xml b/indexer/pom.xml index 2dc8a7e9f10..edecf29bc3f 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/merger/pom.xml b/merger/pom.xml index c359719ddc6..fe1aa8e260f 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/pom.xml b/pom.xml index 98f73fd5a05..be7b8a0f512 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.10-SNAPSHOT + 0.4.10 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 7877aac9f71..eef4e472ae0 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/server/pom.xml b/server/pom.xml index 514059b3fc6..72aa9111551 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 diff --git a/services/pom.xml b/services/pom.xml index 1a7126c6210..215f4e7943a 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.10-SNAPSHOT + 0.4.10 com.metamx druid - 0.4.10-SNAPSHOT + 0.4.10 From 82c4cd5641c9fd24184c7e6e5bfb56e5f0879fd3 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 11:08:39 -0700 Subject: [PATCH 12/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 6bd0b1693c7..2d782de0e3c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 6a9ddf1c915..211f9a60417 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 34e38e481a5..ec19731c69a 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index f67f941af27..f9bcc6e8b4c 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index edecf29bc3f..44b7ce8fd3f 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index fe1aa8e260f..ca9ed3b6df9 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/pom.xml b/pom.xml index be7b8a0f512..a944656a2d3 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.10 + 0.4.11-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index eef4e472ae0..d3a4eb9c523 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 72aa9111551..1891ada95a5 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 215f4e7943a..a44380a2534 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.10 + 0.4.11-SNAPSHOT com.metamx druid - 0.4.10 + 0.4.11-SNAPSHOT From 53c3af6633fd40cdb2aa039d3e29541031fecf2d Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 12:37:21 -0700 Subject: [PATCH 13/28] actually fix the bug mentioned in the last commit --- .../metamx/druid/merger/coordinator/RemoteTaskRunner.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java index fe0577a8c77..9c21ce43d6f 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java @@ -25,6 +25,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.base.Stopwatch; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.MinMaxPriorityQueue; @@ -513,10 +514,12 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider // Syncing state with Zookeeper - don't assign new tasks until the task we just assigned is actually running // on a worker - this avoids overflowing a worker with tasks + Stopwatch timeoutStopwatch = new Stopwatch(); + timeoutStopwatch.start(); synchronized (statusLock) { while (!isWorkerRunningTask(theWorker.getHost(), task.getId())) { statusLock.wait(config.getTaskAssignmentTimeoutDuration().getMillis()); - if (!isWorkerRunningTask(theWorker.getHost(), task.getId())) { + if (timeoutStopwatch.elapsedMillis() >= config.getTaskAssignmentTimeoutDuration().getMillis()) { log.error( "Something went wrong! %s never ran task %s after %s!", theWorker.getHost(), From 57e1cf5f1d9b036a89f5457f90f7657ec333f5a5 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 12:39:01 -0700 Subject: [PATCH 14/28] [maven-release-plugin] prepare release druid-0.4.11 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 2d782de0e3c..f13e5ea7537 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/common/pom.xml b/common/pom.xml index 211f9a60417..8911cf572c9 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/examples/pom.xml b/examples/pom.xml index ec19731c69a..7ce34e5367d 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/index-common/pom.xml b/index-common/pom.xml index f9bcc6e8b4c..f9d3b8ee858 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/indexer/pom.xml b/indexer/pom.xml index 44b7ce8fd3f..3dff0711148 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/merger/pom.xml b/merger/pom.xml index ca9ed3b6df9..aa23a30334f 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/pom.xml b/pom.xml index a944656a2d3..3ae091fab43 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.11-SNAPSHOT + 0.4.11 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index d3a4eb9c523..25e5c44c793 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/server/pom.xml b/server/pom.xml index 1891ada95a5..0bc400b69b1 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 diff --git a/services/pom.xml b/services/pom.xml index a44380a2534..cca9dd8fd90 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.11-SNAPSHOT + 0.4.11 com.metamx druid - 0.4.11-SNAPSHOT + 0.4.11 From 2c45f3349751063ac564909ec5092ff161299bff Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 12:39:09 -0700 Subject: [PATCH 15/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index f13e5ea7537..f137c566e3e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 8911cf572c9..023a382d648 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 7ce34e5367d..f9f4bd85d63 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index f9d3b8ee858..133fed0e613 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 3dff0711148..66f86ca9ef5 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index aa23a30334f..44b9d4fa8bc 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/pom.xml b/pom.xml index 3ae091fab43..ff5f6184ca6 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.11 + 0.4.12-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 25e5c44c793..147d71859b6 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 0bc400b69b1..a1447db0c1a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index cca9dd8fd90..88c0d0fce88 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.11 + 0.4.12-SNAPSHOT com.metamx druid - 0.4.11 + 0.4.12-SNAPSHOT From 5c9df858831f5ada748b5001320b4b334a938ec6 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 15:21:45 -0700 Subject: [PATCH 16/28] better path validation --- .../merger/coordinator/RemoteTaskRunner.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java index 9c21ce43d6f..47f53097022 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/RemoteTaskRunner.java @@ -498,16 +498,15 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogProvider throw new ISE("Length of raw bytes for task too large[%,d > %,d]", rawBytes.length, config.getMaxNumBytes()); } - cf.create() - .withMode(CreateMode.EPHEMERAL) - .forPath( - JOINER.join( - config.getIndexerTaskPath(), - theWorker.getHost(), - task.getId() - ), - rawBytes - ); + String taskPath = JOINER.join(config.getIndexerTaskPath(), theWorker.getHost(), task.getId()); + + if (cf.checkExists().forPath(taskPath) == null) { + cf.create() + .withMode(CreateMode.EPHEMERAL) + .forPath( + taskPath, rawBytes + ); + } runningTasks.put(task.getId(), pendingTasks.remove(task.getId())); log.info("Task %s switched from pending to running", task.getId()); From d8d7d92f6588f38c2c2821ffef184ebc6d81b733 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 17:31:02 -0700 Subject: [PATCH 17/28] [maven-release-plugin] prepare release druid-0.4.12 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index f137c566e3e..15d329f6ba8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/common/pom.xml b/common/pom.xml index 023a382d648..0b42cb2fe99 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/examples/pom.xml b/examples/pom.xml index f9f4bd85d63..fdbd49eb7c1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/index-common/pom.xml b/index-common/pom.xml index 133fed0e613..9edd97d03c3 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/indexer/pom.xml b/indexer/pom.xml index 66f86ca9ef5..274c977340a 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/merger/pom.xml b/merger/pom.xml index 44b9d4fa8bc..55611b2ffda 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/pom.xml b/pom.xml index ff5f6184ca6..f393e18cf46 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12-SNAPSHOT + 0.4.12 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 147d71859b6..9d9cec1fba6 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/server/pom.xml b/server/pom.xml index a1447db0c1a..7445bfb9593 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 diff --git a/services/pom.xml b/services/pom.xml index 88c0d0fce88..90b91644556 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12-SNAPSHOT + 0.4.12 com.metamx druid - 0.4.12-SNAPSHOT + 0.4.12 From 7d34710edf8196d9e3a843e99c9ea09154b05858 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 13 May 2013 17:31:11 -0700 Subject: [PATCH 18/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 15d329f6ba8..93dad68d81f 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 0b42cb2fe99..9758c0822df 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index fdbd49eb7c1..cb3070afda0 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 9edd97d03c3..a0723e20af0 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 274c977340a..546a7eb21ff 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index 55611b2ffda..fcbd7f29f3e 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/pom.xml b/pom.xml index f393e18cf46..00b7f403c1c 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12 + 0.4.13-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 9d9cec1fba6..c12f4bd0d84 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 7445bfb9593..194991b7c56 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 90b91644556..0c2c9d7dc66 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12 + 0.4.13-SNAPSHOT com.metamx druid - 0.4.12 + 0.4.13-SNAPSHOT From 9ed80df03ebc85074a4d7b3e4bed76949d763e4b Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 13:01:53 -0700 Subject: [PATCH 19/28] Allow variable maxPartitionSize in hadoop indexer --- .../druid/indexer/DeterminePartitionsJob.java | 3 +- .../indexer/HadoopDruidIndexerConfig.java | 7 ++- .../indexer/partitions/PartitionsSpec.java | 14 +++++ .../indexer/HadoopDruidIndexerConfigTest.java | 58 +++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/indexer/src/main/java/com/metamx/druid/indexer/DeterminePartitionsJob.java b/indexer/src/main/java/com/metamx/druid/indexer/DeterminePartitionsJob.java index f34ff2988f2..425b33cedff 100644 --- a/indexer/src/main/java/com/metamx/druid/indexer/DeterminePartitionsJob.java +++ b/indexer/src/main/java/com/metamx/druid/indexer/DeterminePartitionsJob.java @@ -504,7 +504,6 @@ public class DeterminePartitionsJob implements Jobby public static class DeterminePartitionsDimSelectionReducer extends DeterminePartitionsDimSelectionBaseReducer { private static final double SHARD_COMBINE_THRESHOLD = 0.25; - private static final double SHARD_OVERSIZE_THRESHOLD = 1.5; private static final int HIGH_CARDINALITY_THRESHOLD = 3000000; @Override @@ -672,7 +671,7 @@ public class DeterminePartitionsJob implements Jobby // Make sure none of these shards are oversized boolean oversized = false; for(final DimPartition partition : dimPartitions.partitions) { - if(partition.rows > config.getTargetPartitionSize() * SHARD_OVERSIZE_THRESHOLD) { + if(partition.rows > config.getMaxPartitionSize()) { log.info("Dimension[%s] has an oversized shard: %s", dimPartitions.dim, partition.shardSpec); oversized = true; } diff --git a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerConfig.java b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerConfig.java index 1dfad9de181..364b880518c 100644 --- a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerConfig.java @@ -236,7 +236,7 @@ public class HadoopDruidIndexerConfig this.partitionsSpec = partitionsSpec; } else { // Backwards compatibility - this.partitionsSpec = new PartitionsSpec(partitionDimension, targetPartitionSize, false); + this.partitionsSpec = new PartitionsSpec(partitionDimension, targetPartitionSize, null, false); } if(granularitySpec != null) { @@ -431,6 +431,11 @@ public class HadoopDruidIndexerConfig return partitionsSpec.getTargetPartitionSize(); } + public long getMaxPartitionSize() + { + return partitionsSpec.getMaxPartitionSize(); + } + public boolean isUpdaterJobSpecSet() { return (updaterJobSpec != null); diff --git a/indexer/src/main/java/com/metamx/druid/indexer/partitions/PartitionsSpec.java b/indexer/src/main/java/com/metamx/druid/indexer/partitions/PartitionsSpec.java index e30bad393f6..5571422585c 100644 --- a/indexer/src/main/java/com/metamx/druid/indexer/partitions/PartitionsSpec.java +++ b/indexer/src/main/java/com/metamx/druid/indexer/partitions/PartitionsSpec.java @@ -8,22 +8,30 @@ import javax.annotation.Nullable; public class PartitionsSpec { + private static final double DEFAULT_OVERSIZE_THRESHOLD = 1.5; + @Nullable private final String partitionDimension; private final long targetPartitionSize; + private final long maxPartitionSize; + private final boolean assumeGrouped; @JsonCreator public PartitionsSpec( @JsonProperty("partitionDimension") @Nullable String partitionDimension, @JsonProperty("targetPartitionSize") @Nullable Long targetPartitionSize, + @JsonProperty("maxPartitionSize") @Nullable Long maxPartitionSize, @JsonProperty("assumeGrouped") @Nullable Boolean assumeGrouped ) { this.partitionDimension = partitionDimension; this.targetPartitionSize = targetPartitionSize == null ? -1 : targetPartitionSize; + this.maxPartitionSize = maxPartitionSize == null + ? (long) (this.targetPartitionSize * DEFAULT_OVERSIZE_THRESHOLD) + : maxPartitionSize; this.assumeGrouped = assumeGrouped == null ? false : assumeGrouped; } @@ -46,6 +54,12 @@ public class PartitionsSpec return targetPartitionSize; } + @JsonProperty + public long getMaxPartitionSize() + { + return maxPartitionSize; + } + @JsonProperty public boolean isAssumeGrouped() { diff --git a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java index 5fdff8ce8b8..87ee95fbfb0 100644 --- a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -248,6 +248,12 @@ public class HadoopDruidIndexerConfigTest 100 ); + Assert.assertEquals( + "getMaxPartitionSize", + partitionsSpec.getMaxPartitionSize(), + 150 + ); + Assert.assertEquals( "getPartitionDimension", partitionsSpec.getPartitionDimension(), @@ -285,6 +291,58 @@ public class HadoopDruidIndexerConfigTest 100 ); + Assert.assertEquals( + "getMaxPartitionSize", + partitionsSpec.getMaxPartitionSize(), + 150 + ); + + Assert.assertEquals( + "getPartitionDimension", + partitionsSpec.getPartitionDimension(), + "foo" + ); + } + + @Test + public void testPartitionsSpecMaxPartitionSize() { + final HadoopDruidIndexerConfig cfg; + + try { + cfg = jsonMapper.readValue( + "{" + + "\"partitionsSpec\":{" + + " \"targetPartitionSize\":100," + + " \"maxPartitionSize\":200," + + " \"partitionDimension\":\"foo\"" + + " }" + + "}", + HadoopDruidIndexerConfig.class + ); + } catch(Exception e) { + throw Throwables.propagate(e); + } + + final PartitionsSpec partitionsSpec = cfg.getPartitionsSpec(); + + Assert.assertEquals( + "isDeterminingPartitions", + partitionsSpec.isDeterminingPartitions(), + true + ); + + Assert.assertEquals( + "getTargetPartitionSize", + partitionsSpec.getTargetPartitionSize(), + 100 + ); + + Assert.assertEquals( + "getMaxPartitionSize", + partitionsSpec.getMaxPartitionSize(), + 200 + ); + Assert.assertEquals( "getPartitionDimension", partitionsSpec.getPartitionDimension(), From 55a986bc7d1cfd7fc4be62f77a4212ac36d51295 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 13:59:07 -0700 Subject: [PATCH 20/28] [maven-release-plugin] prepare release druid-0.4.13 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 93dad68d81f..fe7a9249e97 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/common/pom.xml b/common/pom.xml index 9758c0822df..1cf0c239e43 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/examples/pom.xml b/examples/pom.xml index cb3070afda0..ec073c4b897 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/index-common/pom.xml b/index-common/pom.xml index a0723e20af0..35d74e8d00a 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/indexer/pom.xml b/indexer/pom.xml index 546a7eb21ff..4ac68d18482 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/merger/pom.xml b/merger/pom.xml index fcbd7f29f3e..c08e464dab0 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/pom.xml b/pom.xml index 00b7f403c1c..116f8c70c32 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.13-SNAPSHOT + 0.4.13 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index c12f4bd0d84..e8c9e4eeef7 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/server/pom.xml b/server/pom.xml index 194991b7c56..d41cb2dae2e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 diff --git a/services/pom.xml b/services/pom.xml index 0c2c9d7dc66..8f4ebfad673 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.13-SNAPSHOT + 0.4.13 com.metamx druid - 0.4.13-SNAPSHOT + 0.4.13 From 89990808012e0fe9b5c789253f48e697b186a792 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 13:59:17 -0700 Subject: [PATCH 21/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index fe7a9249e97..4fe90b62759 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 1cf0c239e43..5e4d098a33a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index ec073c4b897..bd3d0c0dae3 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 35d74e8d00a..991841734c2 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 4ac68d18482..b163bcae19a 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index c08e464dab0..4a210f84ec1 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/pom.xml b/pom.xml index 116f8c70c32..55b7356ef3e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.13 + 0.4.14-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index e8c9e4eeef7..ab5709fa7cd 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index d41cb2dae2e..00bbac086f5 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 8f4ebfad673..6574613c3f5 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.13 + 0.4.14-SNAPSHOT com.metamx druid - 0.4.13 + 0.4.14-SNAPSHOT From 343af872b5586d1fc10b75733a9af57be7c365ab Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 14:39:29 -0700 Subject: [PATCH 22/28] DbConnectorConfig: Serialization is hard --- .../java/com/metamx/druid/db/DbConnector.java | 2 +- .../metamx/druid/db/DbConnectorConfig.java | 2 +- .../indexer/HadoopDruidIndexerConfigTest.java | 57 +++++++++++++++---- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/common/src/main/java/com/metamx/druid/db/DbConnector.java b/common/src/main/java/com/metamx/druid/db/DbConnector.java index 1e73b731353..cb202b1a8f2 100644 --- a/common/src/main/java/com/metamx/druid/db/DbConnector.java +++ b/common/src/main/java/com/metamx/druid/db/DbConnector.java @@ -181,7 +181,7 @@ public class DbConnector dataSource.setPassword(config.getDatabasePassword()); dataSource.setUrl(config.getDatabaseConnectURI()); - if (config.isValidationQuery()) { + if (config.useValidationQuery()) { dataSource.setValidationQuery(config.getValidationQuery()); dataSource.setTestOnBorrow(true); } diff --git a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java index fb7d1a99916..e17302a3183 100644 --- a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java +++ b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java @@ -44,7 +44,7 @@ public abstract class DbConnectorConfig @JsonProperty("useValidationQuery") @Config("druid.database.validation") - public boolean isValidationQuery() { + public boolean useValidationQuery() { return false; } diff --git a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java index 87ee95fbfb0..73dcd252055 100644 --- a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.metamx.druid.indexer.granularity.UniformGranularitySpec; import com.metamx.druid.indexer.partitions.PartitionsSpec; +import com.metamx.druid.indexer.updater.DbUpdaterJobSpec; import com.metamx.druid.jackson.DefaultObjectMapper; import org.joda.time.Interval; @@ -39,8 +40,8 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( - "{" + cfg = jsonReadWriteRead( + "{" + " \"granularitySpec\":{" + " \"type\":\"uniform\"," + " \"gran\":\"hour\"," @@ -74,7 +75,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"segmentGranularity\":\"day\"," + "\"intervals\":[\"2012-02-01/P1D\"]" @@ -137,7 +138,7 @@ public class HadoopDruidIndexerConfigTest public void testInvalidGranularityCombination() { boolean thrown = false; try { - final HadoopDruidIndexerConfig cfg = jsonMapper.readValue( + final HadoopDruidIndexerConfig cfg = jsonReadWriteRead( "{" + "\"segmentGranularity\":\"day\"," + "\"intervals\":[\"2012-02-01/P1D\"]," @@ -161,7 +162,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{}", HadoopDruidIndexerConfig.class ); @@ -183,7 +184,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"partitionsSpec\":{" + " \"targetPartitionSize\":100" @@ -221,7 +222,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"partitionsSpec\":{" + " \"targetPartitionSize\":100," @@ -266,7 +267,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"targetPartitionSize\":100," + "\"partitionDimension\":\"foo\"" @@ -309,7 +310,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"partitionsSpec\":{" + " \"targetPartitionSize\":100," @@ -354,7 +355,7 @@ public class HadoopDruidIndexerConfigTest public void testInvalidPartitionsCombination() { boolean thrown = false; try { - final HadoopDruidIndexerConfig cfg = jsonMapper.readValue( + final HadoopDruidIndexerConfig cfg = jsonReadWriteRead( "{" + "\"targetPartitionSize\":100," + "\"partitionsSpec\":{" @@ -369,4 +370,40 @@ public class HadoopDruidIndexerConfigTest Assert.assertTrue("Exception thrown", thrown); } + + @Test + public void testDbUpdaterJobSpec() throws Exception + { + final HadoopDruidIndexerConfig cfg; + + cfg = jsonReadWriteRead( + "{" + + "\"updaterJobSpec\":{\n" + + " \"type\" : \"db\",\n" + + " \"connectURI\" : \"jdbc:mysql://localhost/druid\",\n" + + " \"user\" : \"rofl\",\n" + + " \"password\" : \"p4ssw0rd\",\n" + + " \"segmentTable\" : \"segments\"\n" + + " }" + + "}", + HadoopDruidIndexerConfig.class + ); + + final DbUpdaterJobSpec spec = (DbUpdaterJobSpec) cfg.getUpdaterJobSpec(); + Assert.assertEquals("segments", spec.getSegmentTable()); + Assert.assertEquals("jdbc:mysql://localhost/druid", spec.getDatabaseConnectURI()); + Assert.assertEquals("rofl", spec.getDatabaseUser()); + Assert.assertEquals("p4ssw0rd", spec.getDatabasePassword()); + Assert.assertEquals(false, spec.useValidationQuery()); + } + + private T jsonReadWriteRead(String s, Class klass) + { + try { + return jsonMapper.readValue(jsonMapper.writeValueAsBytes(jsonMapper.readValue(s, klass)), klass); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } } From ecf52f4265f508dd5e973abdc83191f237145edd Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 14:43:29 -0700 Subject: [PATCH 23/28] [maven-release-plugin] prepare release druid-0.4.14 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 4fe90b62759..bfc8b485908 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/common/pom.xml b/common/pom.xml index 5e4d098a33a..86323437154 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/examples/pom.xml b/examples/pom.xml index bd3d0c0dae3..1cf47312167 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/index-common/pom.xml b/index-common/pom.xml index 991841734c2..3d48554f0e3 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/indexer/pom.xml b/indexer/pom.xml index b163bcae19a..46bc15423de 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/merger/pom.xml b/merger/pom.xml index 4a210f84ec1..d9a4422756e 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/pom.xml b/pom.xml index 55b7356ef3e..ed48430a6e2 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.14-SNAPSHOT + 0.4.14 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index ab5709fa7cd..f10fe8dc976 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/server/pom.xml b/server/pom.xml index 00bbac086f5..4c5a42032ee 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 diff --git a/services/pom.xml b/services/pom.xml index 6574613c3f5..c5f181c533a 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.14-SNAPSHOT + 0.4.14 com.metamx druid - 0.4.14-SNAPSHOT + 0.4.14 From 83fa572ab34b54b82d0e318b3bec3f36985946c2 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 14:43:38 -0700 Subject: [PATCH 24/28] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index bfc8b485908..c5b28662317 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 86323437154..2fa1c0d21bd 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 1cf47312167..4ec32589df5 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 3d48554f0e3..eb8cbbe5b4c 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 46bc15423de..917babbf034 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index d9a4422756e..48b843a3cb6 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/pom.xml b/pom.xml index ed48430a6e2..27afe122406 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.14 + 0.4.15-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index f10fe8dc976..146b338c413 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 4c5a42032ee..fa78634de9c 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index c5f181c533a..e09722ff99d 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.14 + 0.4.15-SNAPSHOT com.metamx druid - 0.4.14 + 0.4.15-SNAPSHOT From e8ab0893ff3ffaabc0864e0b026b051cc726b9b6 Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 14 May 2013 14:52:38 -0700 Subject: [PATCH 25/28] fix column size reporting --- .../com/metamx/druid/index/column/ColumnDescriptor.java | 4 ++-- .../com/metamx/druid/index/serde/ColumnPartSerde.java | 2 +- .../metamx/druid/index/serde/ComplexColumnPartSerde.java | 2 +- .../index/serde/DictionaryEncodedColumnPartSerde.java | 6 +++--- .../druid/index/serde/FloatGenericColumnPartSerde.java | 2 +- .../druid/index/serde/LongGenericColumnPartSerde.java | 2 +- .../druid/index/v1/CompressedFloatsIndexedSupplier.java | 2 +- .../druid/index/v1/CompressedLongsIndexedSupplier.java | 2 +- .../src/main/java/com/metamx/druid/index/v1/IndexIO.java | 2 +- .../src/main/java/com/metamx/druid/kv/GenericIndexed.java | 2 +- ...orServiceTaskRunner.java => ThreadPoolTaskRunner.java} | 8 +++----- .../metamx/druid/merger/worker/executor/ExecutorNode.java | 8 ++++---- .../druid/merger/coordinator/RemoteTaskRunnerTest.java | 2 +- .../druid/merger/coordinator/TaskLifecycleTest.java | 2 +- .../java/com/metamx/druid/master/DruidMasterConfig.java | 4 ++-- 15 files changed, 24 insertions(+), 26 deletions(-) rename merger/src/main/java/com/metamx/druid/merger/coordinator/{ExecutorServiceTaskRunner.java => ThreadPoolTaskRunner.java} (95%) diff --git a/index-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java b/index-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java index 2f8281250bd..72918aee9fa 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java @@ -74,9 +74,9 @@ public class ColumnDescriptor return parts; } - public int numBytes() + public long numBytes() { - int retVal = 0; + long retVal = 0; for (ColumnPartSerde part : parts) { retVal += part.numBytes(); diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java index 5cf1f54786b..556fab5ea4d 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java @@ -38,7 +38,7 @@ import java.nio.channels.WritableByteChannel; }) public interface ColumnPartSerde { - public int numBytes(); + public long numBytes(); public void write(WritableByteChannel channel) throws IOException; public ColumnPartSerde read(ByteBuffer buffer, ColumnBuilder builder); } diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java index b5ba4456e03..03e13cfe371 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java @@ -61,7 +61,7 @@ public class ComplexColumnPartSerde implements ColumnPartSerde } @Override - public int numBytes() + public long numBytes() { return column.getSerializedSize(); } diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java index 5f7a38014d6..8f5b80af85c 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java @@ -49,7 +49,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde private final VSizeIndexed multiValuedColumn; private final GenericIndexed bitmaps; - private final int size; + private final long size; public DictionaryEncodedColumnPartSerde( GenericIndexed dictionary, @@ -63,7 +63,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde this.multiValuedColumn = multiValCol; this.bitmaps = bitmaps; - int size = dictionary.getSerializedSize(); + long size = dictionary.getSerializedSize(); if (singleValCol != null && multiValCol == null) { size += singleValCol.getSerializedSize(); } @@ -94,7 +94,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde } @Override - public int numBytes() + public long numBytes() { return 1 + size; } diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java index 85646b0299f..13f86775f30 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java @@ -58,7 +58,7 @@ public class FloatGenericColumnPartSerde implements ColumnPartSerde } @Override - public int numBytes() + public long numBytes() { return compressedFloats.getSerializedSize(); } diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java index 122297cac81..e37f08dff81 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java @@ -58,7 +58,7 @@ public class LongGenericColumnPartSerde implements ColumnPartSerde } @Override - public int numBytes() + public long numBytes() { return compressedLongs.getSerializedSize(); } diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java b/index-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java index 1def2af031f..c93d09d4725 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java @@ -150,7 +150,7 @@ public class CompressedFloatsIndexedSupplier implements Supplier }; } - public int getSerializedSize() + public long getSerializedSize() { return baseFloatBuffers.getSerializedSize() + 1 + 4 + 4; } diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java b/index-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java index d66b68a0c3d..ee752e4a20f 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java @@ -161,7 +161,7 @@ public class CompressedLongsIndexedSupplier implements Supplier }; } - public int getSerializedSize() + public long getSerializedSize() { return baseLongBuffers.getSerializedSize() + 1 + 4 + 4; } diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java b/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java index df3836d7945..ba141e2f819 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java @@ -630,7 +630,7 @@ public class IndexIO GenericIndexed cols = GenericIndexed.fromIterable(columns, GenericIndexed.stringStrategy); - final int numBytes = cols.getSerializedSize() + dims9.getSerializedSize() + 16; + final long numBytes = cols.getSerializedSize() + dims9.getSerializedSize() + 16; final SmooshedWriter writer = v9Smoosher.addWithSmooshedWriter("index.drd", numBytes); cols.writeToChannel(writer); dims9.writeToChannel(writer); diff --git a/index-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java b/index-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java index 5fed0c13b1f..99e003e844c 100644 --- a/index-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java +++ b/index-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java @@ -207,7 +207,7 @@ public class GenericIndexed implements Indexed return -(minIndex + 1); } - public int getSerializedSize() + public long getSerializedSize() { return theBuffer.remaining() + 2 + 4 + 4; } diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/ExecutorServiceTaskRunner.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/ThreadPoolTaskRunner.java similarity index 95% rename from merger/src/main/java/com/metamx/druid/merger/coordinator/ExecutorServiceTaskRunner.java rename to merger/src/main/java/com/metamx/druid/merger/coordinator/ThreadPoolTaskRunner.java index 23fdf851c7d..2906c0196a8 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/ExecutorServiceTaskRunner.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/ThreadPoolTaskRunner.java @@ -28,7 +28,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.druid.Query; import com.metamx.druid.merger.common.TaskStatus; @@ -51,20 +50,19 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; /** * Runs tasks in a JVM thread using an ExecutorService. */ -public class ExecutorServiceTaskRunner implements TaskRunner, QuerySegmentWalker +public class ThreadPoolTaskRunner implements TaskRunner, QuerySegmentWalker { private final TaskToolboxFactory toolboxFactory; private final ListeningExecutorService exec; private final Set runningItems = new ConcurrentSkipListSet(); - private static final EmittingLogger log = new EmittingLogger(ExecutorServiceTaskRunner.class); + private static final EmittingLogger log = new EmittingLogger(ThreadPoolTaskRunner.class); - public ExecutorServiceTaskRunner( + public ThreadPoolTaskRunner( TaskToolboxFactory toolboxFactory, ExecutorService exec ) diff --git a/merger/src/main/java/com/metamx/druid/merger/worker/executor/ExecutorNode.java b/merger/src/main/java/com/metamx/druid/merger/worker/executor/ExecutorNode.java index 66ee23eb0df..93d7b770372 100644 --- a/merger/src/main/java/com/metamx/druid/merger/worker/executor/ExecutorNode.java +++ b/merger/src/main/java/com/metamx/druid/merger/worker/executor/ExecutorNode.java @@ -59,7 +59,7 @@ import com.metamx.druid.merger.common.config.TaskConfig; import com.metamx.druid.merger.common.index.EventReceiverFirehoseFactory; import com.metamx.druid.merger.common.index.ChatHandlerProvider; import com.metamx.druid.merger.common.index.StaticS3FirehoseFactory; -import com.metamx.druid.merger.coordinator.ExecutorServiceTaskRunner; +import com.metamx.druid.merger.coordinator.ThreadPoolTaskRunner; import com.metamx.druid.merger.worker.config.ChatHandlerProviderConfig; import com.metamx.druid.merger.worker.config.WorkerConfig; import com.metamx.druid.utils.PropUtils; @@ -118,7 +118,7 @@ public class ExecutorNode extends BaseServerNode private ServiceAnnouncer serviceAnnouncer = null; private ServiceProvider coordinatorServiceProvider = null; private Server server = null; - private ExecutorServiceTaskRunner taskRunner = null; + private ThreadPoolTaskRunner taskRunner = null; private ExecutorLifecycle executorLifecycle = null; private ChatHandlerProvider chatHandlerProvider = null; @@ -247,7 +247,7 @@ public class ExecutorNode extends BaseServerNode executorLifecycle.join(); } - public ExecutorServiceTaskRunner getTaskRunner() + public ThreadPoolTaskRunner getTaskRunner() { return taskRunner; } @@ -414,7 +414,7 @@ public class ExecutorNode extends BaseServerNode { if (taskRunner == null) { this.taskRunner = lifecycle.addManagedInstance( - new ExecutorServiceTaskRunner( + new ThreadPoolTaskRunner( taskToolboxFactory, Executors.newSingleThreadExecutor( new ThreadFactoryBuilder() diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java index f97a39d60d3..573e4e1cac8 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java @@ -291,7 +291,7 @@ public class RemoteTaskRunnerTest new PathChildrenCache(cf, String.format("%s/worker1", tasksPath), true), cf, workerCuratorCoordinator, - new ExecutorServiceTaskRunner( + new ThreadPoolTaskRunner( new TaskToolboxFactory( new TaskConfig() { diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/TaskLifecycleTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/TaskLifecycleTest.java index 0134984f6cd..42a556d927c 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/TaskLifecycleTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/TaskLifecycleTest.java @@ -162,7 +162,7 @@ public class TaskLifecycleTest new DefaultObjectMapper() ); - tr = new ExecutorServiceTaskRunner( + tr = new ThreadPoolTaskRunner( tb, Executors.newSingleThreadExecutor() ); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java b/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java index a464152ee90..662569e8485 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java @@ -51,13 +51,13 @@ public abstract class DruidMasterConfig @Config("druid.master.merger.on") public boolean isMergeSegments() { - return true; + return false; } @Config("druid.master.conversion.on") public boolean isConvertSegments() { - return true; + return false; } @Config("druid.master.merger.service") From 8bd19e9d045a3f42ee95a7873a0149170c56353a Mon Sep 17 00:00:00 2001 From: cheddar Date: Tue, 14 May 2013 16:55:43 -0500 Subject: [PATCH 26/28] 1) Take in g9yuayon's pull request, fix merge conflicts and formatting --- .../druid/query/group/GroupByQuery.java | 128 +++++- .../query/group/having/AlwaysHavingSpec.java | 14 + .../query/group/having/AndHavingSpec.java | 75 ++++ .../query/group/having/EqualToHavingSpec.java | 94 +++++ .../group/having/GreaterThanHavingSpec.java | 90 ++++ .../druid/query/group/having/HavingSpec.java | 51 +++ .../group/having/LessThanHavingSpec.java | 89 ++++ .../query/group/having/NotHavingSpec.java | 66 +++ .../query/group/having/OrHavingSpec.java | 67 +++ .../{limit => orderby}/DefaultLimitSpec.java | 84 +++- .../group/{limit => orderby}/LimitSpec.java | 2 +- .../{limit => orderby}/NoopLimitSpec.java | 2 +- .../{limit => orderby}/OrderByColumnSpec.java | 47 ++- .../orderby/OrderedPriorityQueueItems.java | 43 ++ .../druid/query/group/orderby/TopNSorter.java | 40 ++ .../query/group/having/HavingSpecTest.java | 227 ++++++++++ .../query/group/orderby/TopNSorterTest.java | 79 ++++ indexer/pom.xml | 6 - pom.xml | 10 - .../java/com/metamx/druid/TestHelper.java | 16 +- .../query/group/GroupByQueryRunnerTest.java | 395 ++++++++++++++---- 21 files changed, 1492 insertions(+), 133 deletions(-) create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java rename client/src/main/java/com/metamx/druid/query/group/{limit => orderby}/DefaultLimitSpec.java (73%) rename client/src/main/java/com/metamx/druid/query/group/{limit => orderby}/LimitSpec.java (97%) rename client/src/main/java/com/metamx/druid/query/group/{limit => orderby}/NoopLimitSpec.java (93%) rename client/src/main/java/com/metamx/druid/query/group/{limit => orderby}/OrderByColumnSpec.java (70%) create mode 100644 client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java create mode 100644 client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java create mode 100644 client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java create mode 100644 client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java diff --git a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java b/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java index 200e207b867..3c48c4cecbc 100644 --- a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java +++ b/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java @@ -22,11 +22,14 @@ package com.metamx.druid.query.group; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; +import com.google.common.base.Functions; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; import com.metamx.druid.BaseQuery; import com.metamx.druid.Query; import com.metamx.druid.QueryGranularity; @@ -37,13 +40,15 @@ import com.metamx.druid.query.Queries; import com.metamx.druid.query.dimension.DefaultDimensionSpec; import com.metamx.druid.query.dimension.DimensionSpec; import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.group.limit.DefaultLimitSpec; -import com.metamx.druid.query.group.limit.LimitSpec; -import com.metamx.druid.query.group.limit.NoopLimitSpec; -import com.metamx.druid.query.group.limit.OrderByColumnSpec; +import com.metamx.druid.query.group.having.HavingSpec; +import com.metamx.druid.query.group.orderby.DefaultLimitSpec; +import com.metamx.druid.query.group.orderby.LimitSpec; +import com.metamx.druid.query.group.orderby.NoopLimitSpec; +import com.metamx.druid.query.group.orderby.OrderByColumnSpec; import com.metamx.druid.query.segment.LegacySegmentSpec; import com.metamx.druid.query.segment.QuerySegmentSpec; +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -57,6 +62,7 @@ public class GroupByQuery extends BaseQuery } private final LimitSpec limitSpec; + private final HavingSpec havingSpec; private final DimFilter dimFilter; private final QueryGranularity granularity; private final List dimensions; @@ -69,34 +75,88 @@ public class GroupByQuery extends BaseQuery public GroupByQuery( @JsonProperty("dataSource") String dataSource, @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, - @JsonProperty("limitSpec") LimitSpec limitSpec, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @JsonProperty("aggregations") List aggregatorSpecs, @JsonProperty("postAggregations") List postAggregatorSpecs, + @JsonProperty("having") HavingSpec havingSpec, + @JsonProperty("limitSpec") LimitSpec limitSpec, + @JsonProperty("orderBy") LimitSpec orderBySpec, @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); - this.limitSpec = (limitSpec == null) ? new NoopLimitSpec() : limitSpec; this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions == null ? ImmutableList.of() : dimensions; this.aggregatorSpecs = aggregatorSpecs; this.postAggregatorSpecs = postAggregatorSpecs == null ? ImmutableList.of() : postAggregatorSpecs; + this.havingSpec = havingSpec; + this.limitSpec = (limitSpec == null) ? (orderBySpec == null ? new NoopLimitSpec() : orderBySpec) : limitSpec; Preconditions.checkNotNull(this.granularity, "Must specify a granularity"); Preconditions.checkNotNull(this.aggregatorSpecs, "Must specify at least one aggregator"); Queries.verifyAggregations(this.aggregatorSpecs, this.postAggregatorSpecs); - orderByLimitFn = this.limitSpec.build(this.dimensions, this.aggregatorSpecs, this.postAggregatorSpecs); + Function, Sequence> postProcFn = + this.limitSpec.build(this.dimensions, this.aggregatorSpecs, this.postAggregatorSpecs); + + if (havingSpec != null) { + postProcFn = Functions.compose( + new Function, Sequence>() + { + @Override + public Sequence apply(@Nullable Sequence input) + { + return Sequences.filter( + input, + new Predicate() + { + @Override + public boolean apply(@Nullable Row input) + { + return GroupByQuery.this.havingSpec.eval(input); + } + } + ); + } + }, + postProcFn + ); + } + + orderByLimitFn = postProcFn; } - @JsonProperty - public LimitSpec getLimitSpec() + /** + * A private constructor that avoids all of the various state checks. Used by the with*() methods where the checks + * have already passed in order for the object to exist. + */ + private GroupByQuery( + String dataSource, + QuerySegmentSpec querySegmentSpec, + DimFilter dimFilter, + QueryGranularity granularity, + List dimensions, + List aggregatorSpecs, + List postAggregatorSpecs, + HavingSpec havingSpec, + LimitSpec orderBySpec, + Function, Sequence> orderByLimitFn, + Map context + ) { - return limitSpec; + super(dataSource, querySegmentSpec, context); + + this.dimFilter = dimFilter; + this.granularity = granularity; + this.dimensions = dimensions; + this.aggregatorSpecs = aggregatorSpecs; + this.postAggregatorSpecs = postAggregatorSpecs; + this.havingSpec = havingSpec; + this.limitSpec = orderBySpec; + this.orderByLimitFn = orderByLimitFn; } @JsonProperty("filter") @@ -129,6 +189,18 @@ public class GroupByQuery extends BaseQuery return postAggregatorSpecs; } + @JsonProperty("having") + public HavingSpec getHavingSpec() + { + return havingSpec; + } + + @JsonProperty + public LimitSpec getOrderBy() + { + return limitSpec; + } + @Override public boolean hasFilters() { @@ -152,12 +224,14 @@ public class GroupByQuery extends BaseQuery return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), - limitSpec, dimFilter, granularity, dimensions, aggregatorSpecs, postAggregatorSpecs, + havingSpec, + limitSpec, + orderByLimitFn, computeOverridenContext(contextOverride) ); } @@ -168,12 +242,14 @@ public class GroupByQuery extends BaseQuery return new GroupByQuery( getDataSource(), spec, - limitSpec, dimFilter, granularity, dimensions, aggregatorSpecs, postAggregatorSpecs, + havingSpec, + limitSpec, + orderByLimitFn, getContext() ); } @@ -187,6 +263,8 @@ public class GroupByQuery extends BaseQuery private List dimensions; private List aggregatorSpecs; private List postAggregatorSpecs; + private HavingSpec havingSpec; + private Map context; private LimitSpec limitSpec = null; @@ -205,6 +283,9 @@ public class GroupByQuery extends BaseQuery dimensions = builder.dimensions; aggregatorSpecs = builder.aggregatorSpecs; postAggregatorSpecs = builder.postAggregatorSpecs; + havingSpec = builder.havingSpec; + limit = builder.limit; + context = builder.context; } @@ -264,7 +345,7 @@ public class GroupByQuery extends BaseQuery private void ensureFluentLimitsNotSet() { - if (! (limit == Integer.MAX_VALUE && orderByColumnSpecs.isEmpty()) ) { + if (!(limit == Integer.MAX_VALUE && orderByColumnSpecs.isEmpty())) { throw new ISE("Ambiguous build, limit[%s] or columnSpecs[%s] already set.", limit, orderByColumnSpecs); } } @@ -351,6 +432,20 @@ public class GroupByQuery extends BaseQuery return this; } + public Builder setHavingSpec(HavingSpec havingSpec) + { + this.havingSpec = havingSpec; + + return this; + } + + public Builder setLimit(Integer limit) + { + this.limit = limit; + + return this; + } + public Builder copy() { return new Builder(this); @@ -361,20 +456,21 @@ public class GroupByQuery extends BaseQuery final LimitSpec theLimitSpec; if (limitSpec == null) { theLimitSpec = new DefaultLimitSpec(orderByColumnSpecs, limit); - } - else { + } else { theLimitSpec = limitSpec; } return new GroupByQuery( dataSource, querySegmentSpec, - theLimitSpec, dimFilter, granularity, dimensions, aggregatorSpecs, postAggregatorSpecs, + havingSpec, + null, + theLimitSpec, context ); } diff --git a/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java new file mode 100644 index 00000000000..3fa27b99ace --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java @@ -0,0 +1,14 @@ +package com.metamx.druid.query.group.having; + +import com.metamx.druid.input.Row; + +/** + */ +public class AlwaysHavingSpec implements HavingSpec +{ + @Override + public boolean eval(Row row) + { + return true; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java new file mode 100644 index 00000000000..f474bc67c95 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java @@ -0,0 +1,75 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.metamx.druid.input.Row; + +import java.util.List; + +/** + * The logical "and" operator for the "having" clause. + */ +public class AndHavingSpec implements HavingSpec +{ + private List havingSpecs; + + @JsonCreator + public AndHavingSpec(@JsonProperty("havingSpecs") List havingSpecs) + { + this.havingSpecs = havingSpecs == null ? ImmutableList.of() : havingSpecs; + } + + @JsonProperty("havingSpecs") + public List getHavingSpecs() + { + return havingSpecs; + } + + @Override + public boolean eval(Row row) + { + for (HavingSpec havingSpec : havingSpecs) { + if (!havingSpec.eval(row)) { + return false; + } + } + + return true; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AndHavingSpec that = (AndHavingSpec) o; + + if (havingSpecs != null ? !havingSpecs.equals(that.havingSpecs) : that.havingSpecs != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return havingSpecs != null ? havingSpecs.hashCode() : 0; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("AndHavingSpec"); + sb.append("{havingSpecs=").append(havingSpecs); + sb.append('}'); + return sb.toString(); + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java new file mode 100644 index 00000000000..f7e5060e4e9 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java @@ -0,0 +1,94 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.druid.input.Row; + +/** + * The "=" operator in a "having" clause. This is similar to SQL's "having aggregation = value", + * except that in SQL an aggregation is an expression instead of an aggregation name as in Druid. + */ +public class EqualToHavingSpec implements HavingSpec +{ + private String aggregationName; + private Number value; + + @JsonCreator + public EqualToHavingSpec( + @JsonProperty("aggregation") String aggName, + @JsonProperty("value") Number value + ) + { + this.aggregationName = aggName; + this.value = value; + } + + @JsonProperty("value") + public Number getValue() + { + return value; + } + + @JsonProperty("aggregation") + public String getAggregationName() + { + return aggregationName; + } + + @Override + public boolean eval(Row row) + { + float metricValue = row.getFloatMetric(aggregationName); + + return Float.compare(value.floatValue(), metricValue) == 0; + } + + /** + * This method treats internal value as double mainly for ease of test. + */ + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + EqualToHavingSpec that = (EqualToHavingSpec) o; + + if (aggregationName != null ? !aggregationName.equals(that.aggregationName) : that.aggregationName != null) { + return false; + } + + if (value != null && that.value != null) { + return Double.compare(value.doubleValue(), that.value.doubleValue()) == 0; + } + + if (value == null && that.value == null) { + return true; + } + + return false; + } + + @Override + public int hashCode() + { + int result = aggregationName != null ? aggregationName.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("EqualToHavingSpec"); + sb.append("{aggregationName='").append(aggregationName).append('\''); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java new file mode 100644 index 00000000000..a64f5ee5050 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java @@ -0,0 +1,90 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.druid.input.Row; + +/** + * The ">" operator in a "having" clause. This is similar to SQL's "having aggregation > value", + * except that an aggregation in SQL is an expression instead of an aggregation name as in Druid. + */ +public class GreaterThanHavingSpec implements HavingSpec +{ + private String aggregationName; + private Number value; + + @JsonCreator + public GreaterThanHavingSpec( + @JsonProperty("aggregation") String aggName, + @JsonProperty("value") Number value + ) + { + this.aggregationName = aggName; + this.value = value; + } + + @JsonProperty("aggregation") + public String getAggregationName() + { + return aggregationName; + } + + @JsonProperty("value") + public Number getValue() + { + return value; + } + + @Override + public boolean eval(Row row) + { + float metricValue = row.getFloatMetric(aggregationName); + + return Float.compare(metricValue, value.floatValue()) > 0; + } + + /** + * This method treats internal value as double mainly for ease of test. + */ + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + GreaterThanHavingSpec that = (GreaterThanHavingSpec) o; + + if (value != null && that.value != null) { + return Double.compare(value.doubleValue(), that.value.doubleValue()) == 0; + } + + if (value == null && that.value == null) { + return true; + } + + return false; + } + + @Override + public int hashCode() + { + int result = aggregationName != null ? aggregationName.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("GreaterThanHavingSpec"); + sb.append("{aggregationName='").append(aggregationName).append('\''); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java new file mode 100644 index 00000000000..b40ae662534 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java @@ -0,0 +1,51 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.metamx.druid.input.Row; + +/** + * A "having" clause that filters aggregated value. This is similar to SQL's "having" + * clause. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = AlwaysHavingSpec.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "and", value = AndHavingSpec.class), + @JsonSubTypes.Type(name = "or", value = OrHavingSpec.class), + @JsonSubTypes.Type(name = "not", value = NotHavingSpec.class), + @JsonSubTypes.Type(name = "greaterThan", value = GreaterThanHavingSpec.class), + @JsonSubTypes.Type(name = "lessThan", value = LessThanHavingSpec.class), + @JsonSubTypes.Type(name = "equalTo", value = EqualToHavingSpec.class) +}) +public interface HavingSpec +{ + /** + * Evaluates if a given row satisfies the having spec. + * + * @param row A Row of data that may contain aggregated values + * + * @return true if the given row satisfies the having spec. False otherwise. + * + * @see Row + */ + public boolean eval(Row row); + + // Atoms for easy combination, but for now they are mostly useful + // for testing. + /** + * A "having" spec that always evaluates to false + */ + public static final HavingSpec NEVER = new HavingSpec() + { + @Override + public boolean eval(Row row) + { + return false; + } + }; + + /** + * A "having" spec that always evaluates to true + */ + public static final HavingSpec ALWAYS = new AlwaysHavingSpec(); +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java new file mode 100644 index 00000000000..86ff4dfaf9c --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java @@ -0,0 +1,89 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.druid.input.Row; + +/** + * The "<" operator in a "having" clause. This is similar to SQL's "having aggregation < value", + * except that an aggregation in SQL is an expression instead of an aggregation name as in Druid. + */ +public class LessThanHavingSpec implements HavingSpec +{ + private String aggregationName; + private Number value; + + public LessThanHavingSpec + ( + @JsonProperty("aggregation") String aggName, + @JsonProperty("value") Number value + ) + { + this.aggregationName = aggName; + this.value = value; + } + + @JsonProperty("aggregation") + public String getAggregationName() + { + return aggregationName; + } + + @JsonProperty("value") + public Number getValue() + { + return value; + } + + @Override + public boolean eval(Row row) + { + float metricValue = row.getFloatMetric(aggregationName); + + return Float.compare(metricValue, value.floatValue()) < 0; + } + + /** + * This method treats internal value as double mainly for ease of test. + */ + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + LessThanHavingSpec that = (LessThanHavingSpec) o; + + if (value != null && that.value != null) { + return Double.compare(value.doubleValue(), that.value.doubleValue()) == 0; + } + + if (value == null && that.value == null) { + return true; + } + + return false; + } + + @Override + public int hashCode() + { + int result = aggregationName != null ? aggregationName.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("LessThanHavingSpec"); + sb.append("{aggregationName='").append(aggregationName).append('\''); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java new file mode 100644 index 00000000000..9324268fbda --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java @@ -0,0 +1,66 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.druid.input.Row; + +/** + * The logical "not" operator for the "having" clause. + */ +public class NotHavingSpec implements HavingSpec +{ + private HavingSpec havingSpec; + + @JsonCreator + public NotHavingSpec(@JsonProperty("havingSpec") HavingSpec havingSpec) + { + this.havingSpec = havingSpec; + } + + @JsonProperty("havingSpec") + public HavingSpec getHavingSpec() + { + return havingSpec; + } + + @Override + public boolean eval(Row row) + { + return !havingSpec.eval(row); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("NotHavingSpec"); + sb.append("{havingSpec=").append(havingSpec); + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + NotHavingSpec that = (NotHavingSpec) o; + + if (havingSpec != null ? !havingSpec.equals(that.havingSpec) : that.havingSpec != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return havingSpec != null ? havingSpec.hashCode() : 0; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java new file mode 100644 index 00000000000..76c8920182f --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java @@ -0,0 +1,67 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.metamx.druid.input.Row; + +import java.util.List; + +/** + * The logical "or" operator for the "having" clause. + */ +public class OrHavingSpec implements HavingSpec +{ + private List havingSpecs; + + @JsonCreator + public OrHavingSpec(@JsonProperty("havingSpecs") List havingSpecs) { + this.havingSpecs = havingSpecs == null ? ImmutableList.of() : havingSpecs; + } + + @JsonProperty("havingSpecs") + public List getHavingSpecs(){ + return havingSpecs; + } + + @Override + public boolean eval(Row row) + { + for(HavingSpec havingSpec: havingSpecs) { + if(havingSpec.eval(row)){ + return true; + } + } + + return false; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + OrHavingSpec that = (OrHavingSpec) o; + + if (havingSpecs != null ? !havingSpecs.equals(that.havingSpecs) : that.havingSpecs != null) return false; + + return true; + } + + @Override + public int hashCode() + { + return havingSpecs != null ? havingSpecs.hashCode() : 0; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append("OrHavingSpec"); + sb.append("{havingSpecs=").append(havingSpecs); + sb.append('}'); + return sb.toString(); + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/limit/DefaultLimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java similarity index 73% rename from client/src/main/java/com/metamx/druid/query/group/limit/DefaultLimitSpec.java rename to client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java index 545c3798785..66049f925f3 100644 --- a/client/src/main/java/com/metamx/druid/query/group/limit/DefaultLimitSpec.java +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group.limit; +package com.metamx.druid.query.group.orderby; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; @@ -35,6 +36,8 @@ import com.metamx.druid.aggregation.post.PostAggregator; import com.metamx.druid.input.Row; import com.metamx.druid.query.dimension.DimensionSpec; +import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -55,7 +58,7 @@ public class DefaultLimitSpec implements LimitSpec this.columns = (columns == null) ? ImmutableList.of() : columns; this.limit = (limit == null) ? Integer.MAX_VALUE : limit; - Preconditions.checkState(limit > 0, "limit[%s] must be >0", limit); + Preconditions.checkArgument(this.limit > 0, "limit[%s] must be >0", limit); } @JsonProperty @@ -75,20 +78,22 @@ public class DefaultLimitSpec implements LimitSpec List dimensions, List aggs, List postAggs ) { - // Materialize the Comparator first for fast-fail error checking. - final Comparator comparator = makeComparator(dimensions, aggs, postAggs); + if (columns.isEmpty()) { + return new LimitingFn(limit); + } - return new Function, Sequence>() - { - @Override - public Sequence apply(Sequence input) - { - return Sequences.limit(Sequences.sort(input, comparator), limit); - } - }; + // Materialize the Comparator first for fast-fail error checking. + final Ordering ordering = makeComparator(dimensions, aggs, postAggs); + + if (limit == Integer.MAX_VALUE) { + return new SortingFn(ordering); + } + else { + return new TopNFunction(ordering, limit); + } } - private Comparator makeComparator( + private Ordering makeComparator( List dimensions, List aggs, List postAggs ) { @@ -174,4 +179,57 @@ public class DefaultLimitSpec implements LimitSpec ", limit=" + limit + '}'; } + + private static class LimitingFn implements Function, Sequence> + { + private int limit; + + public LimitingFn(int limit) + { + this.limit = limit; + } + + @Override + public Sequence apply( + @Nullable Sequence input + ) + { + return Sequences.limit(input, limit); + } + } + + private static class SortingFn implements Function, Sequence> + { + private final Ordering ordering; + + public SortingFn(Ordering ordering) {this.ordering = ordering;} + + @Override + public Sequence apply(@Nullable Sequence input) + { + return Sequences.sort(input, ordering); + } + } + + private static class TopNFunction implements Function, Sequence> + { + private final TopNSorter sorter; + private final int limit; + + public TopNFunction(Ordering ordering, int limit) + { + this.limit = limit; + + this.sorter = new TopNSorter(ordering); + } + + @Override + public Sequence apply( + @Nullable Sequence input + ) + { + final ArrayList materializedList = Sequences.toList(input, Lists.newArrayList()); + return Sequences.simple(sorter.toTopN(materializedList, limit)); + } + } } diff --git a/client/src/main/java/com/metamx/druid/query/group/limit/LimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java similarity index 97% rename from client/src/main/java/com/metamx/druid/query/group/limit/LimitSpec.java rename to client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java index 4522b4739c2..cee3c76e55d 100644 --- a/client/src/main/java/com/metamx/druid/query/group/limit/LimitSpec.java +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group.limit; +package com.metamx.druid.query.group.orderby; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/client/src/main/java/com/metamx/druid/query/group/limit/NoopLimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/group/limit/NoopLimitSpec.java rename to client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java index f9332f8754d..2493cd4cbf3 100644 --- a/client/src/main/java/com/metamx/druid/query/group/limit/NoopLimitSpec.java +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java @@ -1,4 +1,4 @@ -package com.metamx.druid.query.group.limit; +package com.metamx.druid.query.group.orderby; import com.google.common.base.Function; import com.google.common.base.Functions; diff --git a/client/src/main/java/com/metamx/druid/query/group/limit/OrderByColumnSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java similarity index 70% rename from client/src/main/java/com/metamx/druid/query/group/limit/OrderByColumnSpec.java rename to client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java index ba9aa79fa87..6a02ec962f8 100644 --- a/client/src/main/java/com/metamx/druid/query/group/limit/OrderByColumnSpec.java +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java @@ -1,12 +1,17 @@ -package com.metamx.druid.query.group.limit; +package com.metamx.druid.query.group.orderby; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.metamx.common.IAE; import com.metamx.common.ISE; +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -52,6 +57,46 @@ public class OrderByColumnSpec } } + public static OrderByColumnSpec asc(String dimension) + { + return new OrderByColumnSpec(dimension, Direction.ASCENDING); + } + + public static List ascending(String... dimension) + { + return Lists.transform( + Arrays.asList(dimension), + new Function() + { + @Override + public OrderByColumnSpec apply(@Nullable String input) + { + return asc(input); + } + } + ); + } + + public static OrderByColumnSpec desc(String dimension) + { + return new OrderByColumnSpec(dimension, Direction.DESCENDING); + } + + public static List descending(String... dimension) + { + return Lists.transform( + Arrays.asList(dimension), + new Function() + { + @Override + public OrderByColumnSpec apply(@Nullable String input) + { + return desc(input); + } + } + ); + } + public OrderByColumnSpec( String dimension, Direction direction diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java new file mode 100644 index 00000000000..5b5a3f48cb6 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java @@ -0,0 +1,43 @@ +package com.metamx.druid.query.group.orderby; + +import com.google.common.collect.MinMaxPriorityQueue; + +import java.util.Iterator; + +/** + * Utility class that supports iterating a priority queue in sorted order. + */ +class OrderedPriorityQueueItems implements Iterable +{ + private MinMaxPriorityQueue rows; + + public OrderedPriorityQueueItems(MinMaxPriorityQueue rows) + { + this.rows = rows; + } + + @Override + public Iterator iterator() + { + return new Iterator() { + + @Override + public boolean hasNext() + { + return !rows.isEmpty(); + } + + @Override + public T next() + { + return rows.poll(); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException("Can't remove any item from an intermediary heap for orderBy/limit"); + } + }; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java b/client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java new file mode 100644 index 00000000000..7160ba62074 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java @@ -0,0 +1,40 @@ +package com.metamx.druid.query.group.orderby; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.MinMaxPriorityQueue; +import com.google.common.collect.Ordering; + +/** + * A utility class that sorts a list of comparable items in the given order, and keeps only the + * top N sorted items. + */ +public class TopNSorter +{ + private Ordering ordering; + + /** + * Constructs a sorter that will sort items with given ordering. + * @param ordering the order that this sorter instance will use for sorting + */ + public TopNSorter(Ordering ordering) + { + this.ordering = ordering; + } + + /** + * Sorts a list of rows and retain the top n items + * @param items the collections of items to be sorted + * @param n the number of items to be retained + * @return Top n items that are sorted in the order specified when this instance is constructed. + */ + public Iterable toTopN(Iterable items, int n) + { + if(n <= 0) { + return ImmutableList.of(); + } + + MinMaxPriorityQueue queue = MinMaxPriorityQueue.orderedBy(ordering).maximumSize(n).create(items); + + return new OrderedPriorityQueueItems(queue); + } +} diff --git a/client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java b/client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java new file mode 100644 index 00000000000..28c91878e57 --- /dev/null +++ b/client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java @@ -0,0 +1,227 @@ +package com.metamx.druid.query.group.having; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.druid.input.MapBasedInputRow; +import com.metamx.druid.input.Row; +import com.metamx.druid.jackson.DefaultObjectMapper; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +public class HavingSpecTest +{ + private static final Row ROW = new MapBasedInputRow(0, new ArrayList(), ImmutableMap.of("metric", (Object)Float.valueOf(10))); + + @Test + public void testHavingClauseSerde() throws Exception { + List havings = Arrays.asList( + new GreaterThanHavingSpec("agg", Double.valueOf(1.3)), + new OrHavingSpec( + Arrays.asList( + new LessThanHavingSpec("lessAgg", Long.valueOf(1L)), + new NotHavingSpec(new EqualToHavingSpec("equalAgg", Double.valueOf(2))) + ) + ) + ); + + HavingSpec andHavingSpec = new AndHavingSpec(havings); + + Map notMap = ImmutableMap.of( + "type", "not", + "havingSpec", ImmutableMap.of("type", "equalTo", "aggregation", "equalAgg", "value", 2.0) + ); + + Map lessMap = ImmutableMap.of( + "type", "lessThan", + "aggregation", "lessAgg", + "value", 1 + ); + + Map greaterMap = ImmutableMap.of( + "type", "greaterThan", + "aggregation", "agg", + "value", 1.3 + ); + + Map orMap = ImmutableMap.of( + "type", "or", + "havingSpecs", ImmutableList.of(lessMap, notMap) + ); + + Map payloadMap = ImmutableMap.of( + "type", "and", + "havingSpecs", ImmutableList.of(greaterMap, orMap) + ); + + ObjectMapper mapper = new DefaultObjectMapper(); + assertEquals(andHavingSpec, mapper.convertValue(payloadMap, AndHavingSpec.class)); + } + + @Test + public void testGreaterThanHavingSpec() { + GreaterThanHavingSpec spec = new GreaterThanHavingSpec("metric", 10.003); + assertFalse(spec.eval(ROW)); + + spec = new GreaterThanHavingSpec("metric", 10); + assertFalse(spec.eval(ROW)); + + spec = new GreaterThanHavingSpec("metric", 9); + assertTrue(spec.eval(ROW)); + } + + private MapBasedInputRow makeRow(long ts, String dim, int value) + { + List dimensions = Lists.newArrayList(dim); + Map metrics = ImmutableMap.of("metric", (Object) Float.valueOf(value)); + + return new MapBasedInputRow(ts, dimensions, metrics); + } + + @Test + public void testLessThanHavingSpec() { + LessThanHavingSpec spec = new LessThanHavingSpec("metric", 10); + assertFalse(spec.eval(ROW)); + + spec = new LessThanHavingSpec("metric", 11); + assertTrue(spec.eval(ROW)); + + spec = new LessThanHavingSpec("metric", 9); + assertFalse(spec.eval(ROW)); + } + + @Test + public void testEqualHavingSpec() { + EqualToHavingSpec spec = new EqualToHavingSpec("metric", 10); + assertTrue(spec.eval(ROW)); + + spec = new EqualToHavingSpec("metric", 9); + assertFalse(spec.eval(ROW)); + + spec = new EqualToHavingSpec("metric", 11); + assertFalse(spec.eval(ROW)); + } + + private static class CountingHavingSpec implements HavingSpec { + + private final AtomicInteger counter; + private final boolean value; + private CountingHavingSpec(AtomicInteger counter, boolean value) + { + this.counter = counter; + this.value = value; + } + + @Override + public boolean eval(Row row) + { + counter.incrementAndGet(); + return value; + } + } + + @Test + public void testAndHavingSpecShouldSupportShortcutEvaluation () { + AtomicInteger counter = new AtomicInteger(0); + AndHavingSpec spec = new AndHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, false) + )); + + spec.eval(ROW); + + assertEquals(2, counter.get()); + } + + @Test + public void testAndHavingSpec () { + AtomicInteger counter = new AtomicInteger(0); + AndHavingSpec spec = new AndHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true) + )); + + spec.eval(ROW); + + assertEquals(4, counter.get()); + + counter.set(0); + spec = new AndHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true) + )); + + spec.eval(ROW); + + assertEquals(1, counter.get()); + } + + @Test + public void testOrHavingSpecSupportsShortcutEvaluation() { + AtomicInteger counter = new AtomicInteger(0); + OrHavingSpec spec = new OrHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, true), + new CountingHavingSpec(counter, false) + )); + + spec.eval(ROW); + + assertEquals(1, counter.get()); + } + + @Test + public void testOrHavingSpec () { + AtomicInteger counter = new AtomicInteger(0); + OrHavingSpec spec = new OrHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, false) + )); + + spec.eval(ROW); + + assertEquals(4, counter.get()); + + counter.set(0); + spec = new OrHavingSpec(ImmutableList.of( + (HavingSpec)new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, false), + new CountingHavingSpec(counter, true) + )); + + spec.eval(ROW); + + assertEquals(4, counter.get()); + } + + @Test + public void testNotHavingSepc() { + NotHavingSpec spec = new NotHavingSpec(HavingSpec.NEVER); + assertTrue(spec.eval(ROW)); + + spec = new NotHavingSpec(HavingSpec.ALWAYS); + assertFalse(spec.eval(ROW)); + + } +} diff --git a/client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java b/client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java new file mode 100644 index 00000000000..872155e5049 --- /dev/null +++ b/client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java @@ -0,0 +1,79 @@ +package com.metamx.druid.query.group.orderby; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Random; + + +@RunWith(Parameterized.class) +public class TopNSorterTest +{ + private static final long SEED = 2L; + private static final Ordering ASC = Ordering.natural(); + private static final Ordering DESC = Ordering.natural().reverse(); + + private static final List EMPTY = Collections.EMPTY_LIST; + private static final List SINGLE = Lists.newArrayList("a"); + private static final List RAW_ASC = Lists.newArrayList(Splitter.fixedLength(1).split("abcdefghijk")); + private static final List RAW_DESC = Lists.newArrayList(Splitter.fixedLength(1).split("kjihgfedcba")); + + private Ordering ordering; + private List rawInput; + private int limit; + + @Parameterized.Parameters + public static Collection makeTestData(){ + Object[][] data = new Object[][] { + { ASC, RAW_ASC, RAW_ASC.size() - 2}, + { ASC, RAW_ASC, RAW_ASC.size()}, + { ASC, RAW_ASC, RAW_ASC.size() + 2}, + { ASC, RAW_ASC, 0}, + { ASC, SINGLE, 0}, + { ASC, SINGLE, 1}, + { ASC, SINGLE, 2}, + { ASC, SINGLE, 3}, + { ASC, EMPTY, 0}, + { ASC, EMPTY, 1}, + { DESC, RAW_DESC, RAW_DESC.size() - 2}, + { DESC, RAW_DESC, RAW_DESC.size()}, + { DESC, RAW_DESC, RAW_DESC.size() + 2}, + { DESC, RAW_DESC, 0}, + { DESC, RAW_DESC, 0}, + { DESC, SINGLE, 1}, + { DESC, SINGLE, 2}, + { DESC, SINGLE, 3}, + { DESC, EMPTY, 0}, + { DESC, EMPTY, 1}, + }; + + return Arrays.asList(data); + } + + public TopNSorterTest(Ordering ordering, List rawInput, int limit){ + this.ordering = ordering; + this.rawInput = rawInput; + this.limit = limit; + } + + @Test + public void testOrderByWithLimit() + { + List expected = rawInput.subList(0, Math.min(limit, rawInput.size())); + List inputs = Lists.newArrayList(rawInput); + Collections.shuffle(inputs, new Random(2)); + + Iterable result = new TopNSorter(ordering).toTopN(inputs, limit); + + Assert.assertEquals(expected, Lists.newArrayList(result)); + } +} diff --git a/indexer/pom.xml b/indexer/pom.xml index 546a7eb21ff..d659c479cb6 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -120,12 +120,6 @@ ${project.build.directory}/${project.artifactId}-${project.version}-selfcontained.jar - - - org.codehaus.jackson - druid.org.codehaus.jackson - - diff --git a/pom.xml b/pom.xml index 00b7f403c1c..d01a7e4d1f7 100644 --- a/pom.xml +++ b/pom.xml @@ -209,16 +209,6 @@ jackson-jaxrs-json-provider 2.1.4 - - org.codehaus.jackson - jackson-core-asl - 1.9.11 - - - org.codehaus.jackson - jackson-mapper-asl - 1.9.11 - javax.inject javax.inject diff --git a/server/src/test/java/com/metamx/druid/TestHelper.java b/server/src/test/java/com/metamx/druid/TestHelper.java index 53d45192f7c..8a15a73f724 100644 --- a/server/src/test/java/com/metamx/druid/TestHelper.java +++ b/server/src/test/java/com/metamx/druid/TestHelper.java @@ -69,7 +69,7 @@ public class TestHelper assertResult(failMsg, expectedNext, next); assertResult( - String.format("%sSecond iterator bad, multiple calls to iterator() should be safe", failMsg), + String.format("%s: Second iterator bad, multiple calls to iterator() should be safe", failMsg), expectedNext, next2 ); @@ -77,20 +77,20 @@ public class TestHelper if (resultsIter.hasNext()) { Assert.fail( - String.format("%sExpected resultsIter to be exhausted, next element was %s", failMsg, resultsIter.next()) + String.format("%s: Expected resultsIter to be exhausted, next element was %s", failMsg, resultsIter.next()) ); } if (resultsIter2.hasNext()) { Assert.fail( - String.format("%sExpected resultsIter2 to be exhausted, next element was %s", failMsg, resultsIter.next()) + String.format("%s: Expected resultsIter2 to be exhausted, next element was %s", failMsg, resultsIter.next()) ); } if (expectedResultsIter.hasNext()) { Assert.fail( String.format( - "%sExpected expectedResultsIter to be exhausted, next element was %s", failMsg, expectedResultsIter.next() + "%s: Expected expectedResultsIter to be exhausted, next element was %s", failMsg, expectedResultsIter.next() ) ); } @@ -109,7 +109,7 @@ public class TestHelper Assert.assertEquals(failMsg, expectedNext, next); Assert.assertEquals( - String.format("%sSecond iterator bad, multiple calls to iterator() should be safe", failMsg), + String.format("%s: Second iterator bad, multiple calls to iterator() should be safe", failMsg), expectedNext, next2 ); @@ -117,20 +117,20 @@ public class TestHelper if (resultsIter.hasNext()) { Assert.fail( - String.format("%sExpected resultsIter to be exhausted, next element was %s", failMsg, resultsIter.next()) + String.format("%s: Expected resultsIter to be exhausted, next element was %s", failMsg, resultsIter.next()) ); } if (resultsIter2.hasNext()) { Assert.fail( - String.format("%sExpected resultsIter2 to be exhausted, next element was %s", failMsg, resultsIter.next()) + String.format("%s: Expected resultsIter2 to be exhausted, next element was %s", failMsg, resultsIter.next()) ); } if (expectedResultsIter.hasNext()) { Assert.fail( String.format( - "%sExpected expectedResultsIter to be exhausted, next element was %s", failMsg, expectedResultsIter.next() + "%s: Expected expectedResultsIter to be exhausted, next element was %s", failMsg, expectedResultsIter.next() ) ); } diff --git a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java b/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java index 7dfc83f6f4f..a6e07b52d6e 100644 --- a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java @@ -22,9 +22,12 @@ package com.metamx.druid.query.group; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.druid.PeriodGranularity; @@ -41,7 +44,12 @@ import com.metamx.druid.query.QueryRunner; import com.metamx.druid.query.QueryRunnerTestHelper; import com.metamx.druid.query.dimension.DefaultDimensionSpec; import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.query.group.limit.OrderByColumnSpec; +import com.metamx.druid.query.group.having.EqualToHavingSpec; +import com.metamx.druid.query.group.having.GreaterThanHavingSpec; +import com.metamx.druid.query.group.having.OrHavingSpec; +import com.metamx.druid.query.group.orderby.DefaultLimitSpec; +import com.metamx.druid.query.group.orderby.LimitSpec; +import com.metamx.druid.query.group.orderby.OrderByColumnSpec; import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -56,6 +64,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -69,28 +78,30 @@ public class GroupByQueryRunnerTest public static Collection constructorFeeder() throws IOException { final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( - new GroupByQueryEngine( - new GroupByQueryEngineConfig() - { - @Override - public int getMaxIntermediateRows() - { - return 10000; - } - }, - new StupidPool( - new Supplier() - { - @Override - public ByteBuffer get() - { - return ByteBuffer.allocate(1024 * 1024); - } - } - ) - ), - new GroupByQueryRunnerFactoryConfig(){} - ); + new GroupByQueryEngine( + new GroupByQueryEngineConfig() + { + @Override + public int getMaxIntermediateRows() + { + return 10000; + } + }, + new StupidPool( + new Supplier() + { + @Override + public ByteBuffer get() + { + return ByteBuffer.allocate(1024 * 1024); + } + } + ) + ), + new GroupByQueryRunnerFactoryConfig() + { + } + ); return Lists.newArrayList( Iterables.transform( @@ -106,13 +117,15 @@ public class GroupByQueryRunnerTest ); } - public GroupByQueryRunnerTest(GroupByQueryRunnerFactory factory, QueryRunner runner) { + public GroupByQueryRunnerTest(GroupByQueryRunnerFactory factory, QueryRunner runner) + { this.factory = factory; this.runner = runner; } @Test - public void testGroupBy() { + public void testGroupBy() + { GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -127,35 +140,36 @@ public class GroupByQueryRunnerTest .setGranularity(QueryRunnerTestHelper.dayGran) .build(); - List expectedResults = Arrays.asList( - createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), + List expectedResults = Arrays.asList( + createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), + createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), + createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), + createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), + createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), + createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), + createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) - ); + createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), + createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), + createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), + createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), + createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), + createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), + createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), + createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + ); - Iterable results = Sequences.toList(runner.run(query), Lists.newArrayList()); + Iterable results = Sequences.toList(runner.run(query), Lists.newArrayList()); - TestHelper.assertExpectedObjects(expectedResults, results, ""); + TestHelper.assertExpectedObjects(expectedResults, results, ""); } @Test - public void testGroupByWithTimeZone() { + public void testGroupByWithTimeZone() + { DateTimeZone tz = DateTimeZone.forID("America/Los_Angeles"); GroupByQuery query = GroupByQuery.builder() @@ -187,38 +201,39 @@ public class GroupByQueryRunnerTest ) .build(); - List expectedResults = Arrays.asList( - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "automotive", "rows", 1L, "idx", 135L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "business", "rows", 1L, "idx", 118L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "entertainment", "rows", 1L, "idx", 158L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "health", "rows", 1L, "idx", 120L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "mezzanine", "rows", 3L, "idx", 2870L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "news", "rows", 1L, "idx", 121L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "premium", "rows", 3L, "idx", 2900L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "technology", "rows", 1L, "idx", 78L), - createExpectedRow(new DateTime("2011-03-31", tz), "alias", "travel", "rows", 1L, "idx", 119L), + List expectedResults = Arrays.asList( + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "automotive", "rows", 1L, "idx", 135L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "business", "rows", 1L, "idx", 118L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "entertainment", "rows", 1L, "idx", 158L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "health", "rows", 1L, "idx", 120L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "mezzanine", "rows", 3L, "idx", 2870L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "news", "rows", 1L, "idx", 121L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "premium", "rows", 3L, "idx", 2900L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "technology", "rows", 1L, "idx", 78L), + createExpectedRow(new DateTime("2011-03-31", tz), "alias", "travel", "rows", 1L, "idx", 119L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "automotive", "rows", 1L, "idx", 147L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "business", "rows", 1L, "idx", 112L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "entertainment", "rows", 1L, "idx", 166L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "health", "rows", 1L, "idx", 113L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "mezzanine", "rows", 3L, "idx", 2447L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "news", "rows", 1L, "idx", 114L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "premium", "rows", 3L, "idx", 2505L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "technology", "rows", 1L, "idx", 97L), - createExpectedRow(new DateTime("2011-04-01", tz), "alias", "travel", "rows", 1L, "idx", 126L) - ); + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "automotive", "rows", 1L, "idx", 147L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "business", "rows", 1L, "idx", 112L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "entertainment", "rows", 1L, "idx", 166L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "health", "rows", 1L, "idx", 113L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "mezzanine", "rows", 3L, "idx", 2447L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "news", "rows", 1L, "idx", 114L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "premium", "rows", 3L, "idx", 2505L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "technology", "rows", 1L, "idx", 97L), + createExpectedRow(new DateTime("2011-04-01", tz), "alias", "travel", "rows", 1L, "idx", 126L) + ); - Iterable results = Sequences.toList( - runner.run(query), - Lists.newArrayList() - ); + Iterable results = Sequences.toList( + runner.run(query), + Lists.newArrayList() + ); - TestHelper.assertExpectedObjects(expectedResults, results, ""); + TestHelper.assertExpectedObjects(expectedResults, results, ""); } @Test - public void testMergeResults() { + public void testMergeResults() + { GroupByQuery.Builder builder = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -282,7 +297,180 @@ public class GroupByQueryRunnerTest TestHelper.assertExpectedObjects(allGranExpectedResults, runner.run(allGranQuery), "direct"); TestHelper.assertExpectedObjects(allGranExpectedResults, mergedRunner.run(allGranQuery), "merged"); + } + @Test + public void testMergeResultsWithLimit() + { + for (int limit = 1; limit < 20; ++limit) { + doTestMergeResultsWithValidLimit(limit); + } + } + + private void doTestMergeResultsWithValidLimit(final int limit) + { + GroupByQuery.Builder builder = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setInterval("2011-04-02/2011-04-04") + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setLimit(Integer.valueOf(limit)); + + final GroupByQuery fullQuery = builder.build(); + + List expectedResults = Arrays.asList( + createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + ); + + QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + + mergeRunner.run(fullQuery).accumulate(null, new Accumulator() + { + @Override + public Object accumulate(Object o, Row row) + { + System.out.printf("%d: %s%n", limit, row); + return null; + } + }); + + TestHelper.assertExpectedObjects( + Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery), String.format("limit: %d", limit) + ); + } + + @Test(expected = IllegalArgumentException.class) + public void testMergeResultsWithNegativeLimit() + { + GroupByQuery.Builder builder = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setInterval("2011-04-02/2011-04-04") + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setLimit(Integer.valueOf(-1)); + + builder.build(); + } + + @Test + public void testMergeResultsWithOrderBy() + { + LimitSpec[] orderBySpecs = new LimitSpec[]{ + new DefaultLimitSpec(OrderByColumnSpec.ascending("idx"), null), + new DefaultLimitSpec(OrderByColumnSpec.ascending("rows", "idx"), null), + new DefaultLimitSpec(OrderByColumnSpec.descending("idx"), null), + new DefaultLimitSpec(OrderByColumnSpec.descending("rows", "idx"), null), + }; + + final Comparator idxComparator = + new Comparator() + { + @Override + public int compare(Row o1, Row o2) + { + return Float.compare(o1.getFloatMetric("idx"), o2.getFloatMetric("idx")); + } + }; + + Comparator rowsIdxComparator = + new Comparator() + { + + @Override + public int compare(Row o1, Row o2) + { + int value = Float.compare(o1.getFloatMetric("rows"), o2.getFloatMetric("rows")); + if (value != 0) { + return value; + } + + return idxComparator.compare(o1, o2); + } + }; + + List allResults = Arrays.asList( + createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), + createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), + createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), + createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), + createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), + createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), + createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + ); + + List> expectedResults = Lists.newArrayList( + Ordering.from(idxComparator).sortedCopy(allResults), + Ordering.from(rowsIdxComparator).sortedCopy(allResults), + Ordering.from(idxComparator).reverse().sortedCopy(allResults), + Ordering.from(rowsIdxComparator).reverse().sortedCopy(allResults) + ); + + for (int i = 0; i < orderBySpecs.length; ++i) { + doTestMergeResultsWithOrderBy(orderBySpecs[i], expectedResults.get(i)); + } + } + + private void doTestMergeResultsWithOrderBy(LimitSpec orderBySpec, List expectedResults) + { + GroupByQuery.Builder builder = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setInterval("2011-04-02/2011-04-04") + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setLimitSpec(orderBySpec); + + final GroupByQuery fullQuery = builder.build(); + + QueryRunner mergedRunner = new GroupByQueryQueryToolChest().mergeResults( + new QueryRunner() + { + @Override + public Sequence run(Query query) + { + // simulate two daily segments + final Query query1 = query.withQuerySegmentSpec( + new MultipleIntervalSegmentSpec(Lists.newArrayList(new Interval("2011-04-02/2011-04-03"))) + ); + final Query query2 = query.withQuerySegmentSpec( + new MultipleIntervalSegmentSpec(Lists.newArrayList(new Interval("2011-04-03/2011-04-04"))) + ); + return Sequences.concat(runner.run(query1), runner.run(query2)); + } + } + ); + + TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery), "merged"); } @Test @@ -403,6 +591,59 @@ public class GroupByQueryRunnerTest ); } + @Test + public void testHavingSpec() + { + List expectedResults = Arrays.asList( + createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), + createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), + createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L) + ); + + GroupByQuery.Builder builder = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setInterval("2011-04-02/2011-04-04") + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setHavingSpec( + new OrHavingSpec( + ImmutableList.of( + new GreaterThanHavingSpec("rows", 2L), + new EqualToHavingSpec("idx", 217L) + ) + ) + ); + + final GroupByQuery fullQuery = builder.build(); + + QueryRunner mergedRunner = new GroupByQueryQueryToolChest().mergeResults( + new QueryRunner() + { + @Override + public Sequence run(Query query) + { + // simulate two daily segments + final Query query1 = query.withQuerySegmentSpec( + new MultipleIntervalSegmentSpec(Lists.newArrayList(new Interval("2011-04-02/2011-04-03"))) + ); + final Query query2 = query.withQuerySegmentSpec( + new MultipleIntervalSegmentSpec(Lists.newArrayList(new Interval("2011-04-03/2011-04-04"))) + ); + return Sequences.concat(runner.run(query1), runner.run(query2)); + } + } + ); + + TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery), "merged"); + } + private Row createExpectedRow(final String timestamp, Object... vals) { return createExpectedRow(new DateTime(timestamp), vals); @@ -413,10 +654,10 @@ public class GroupByQueryRunnerTest Preconditions.checkArgument(vals.length % 2 == 0); Map theVals = Maps.newHashMap(); - for (int i = 0; i < vals.length; i+=2) { - theVals.put(vals[i].toString(), vals[i+1]); + for (int i = 0; i < vals.length; i += 2) { + theVals.put(vals[i].toString(), vals[i + 1]); } - return new MapBasedRow(timestamp, theVals); + return new MapBasedRow(new DateTime(timestamp), theVals); } } From fe5627e119e172dcc346428b75dbe363b911d18a Mon Sep 17 00:00:00 2001 From: cheddar Date: Tue, 14 May 2013 16:59:34 -0500 Subject: [PATCH 27/28] 1) Left some log lines in the tests... That's no good. --- .../druid/query/group/GroupByQueryRunnerTest.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java b/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java index a6e07b52d6e..ef1567f0e81 100644 --- a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java @@ -27,7 +27,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; -import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.druid.PeriodGranularity; @@ -339,16 +338,6 @@ public class GroupByQueryRunnerTest QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); - mergeRunner.run(fullQuery).accumulate(null, new Accumulator() - { - @Override - public Object accumulate(Object o, Row row) - { - System.out.printf("%d: %s%n", limit, row); - return null; - } - }); - TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery), String.format("limit: %d", limit) ); From 236fe64e91eaf6602516e240432c3fcf49aabbd0 Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 14 May 2013 15:17:49 -0700 Subject: [PATCH 28/28] fix stray import --- .../main/java/com/metamx/druid/client/DirectDruidClient.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java b/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java index 445ea44dec1..92a4e5b9426 100644 --- a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java +++ b/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java @@ -47,7 +47,6 @@ import com.metamx.http.client.HttpClient; import com.metamx.http.client.io.AppendableByteArrayInputStream; import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; - import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpResponse; @@ -61,7 +60,6 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; /** */