From b420892dd9dc038e5d695fa766228ece04bb16c3 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 17 May 2013 11:08:39 -0700 Subject: [PATCH 01/12] HadoopDruidIndexerJob: Fix NPE --- .../com/metamx/druid/indexer/HadoopDruidIndexerJob.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java index f2f192fbeea..d9dc0bb1e11 100644 --- a/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java +++ b/indexer/src/main/java/com/metamx/druid/indexer/HadoopDruidIndexerJob.java @@ -102,6 +102,10 @@ public class HadoopDruidIndexerJob implements Jobby } } + if (failedMessage == null) { + publishedSegments = IndexGeneratorJob.getPublishedSegments(config); + } + if (!config.isLeaveIntermediate()) { if (failedMessage == null || config.isCleanupOnFailure()) { Path workingPath = config.makeIntermediatePath(); @@ -119,8 +123,6 @@ public class HadoopDruidIndexerJob implements Jobby throw new ISE(failedMessage); } - publishedSegments = IndexGeneratorJob.getPublishedSegments(config); - return true; } From 5df2e40d4e52657c580e702b9702166e5a9f34c1 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 17 May 2013 11:14:08 -0700 Subject: [PATCH 02/12] [maven-release-plugin] prepare release druid-0.4.12.1 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 93dad68d81f..8d1fabd80dd 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/common/pom.xml b/common/pom.xml index 9758c0822df..61e683d79e6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/examples/pom.xml b/examples/pom.xml index cb3070afda0..08eb17f7957 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/index-common/pom.xml b/index-common/pom.xml index a0723e20af0..886f041eac2 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/indexer/pom.xml b/indexer/pom.xml index 546a7eb21ff..26b398984d9 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/merger/pom.xml b/merger/pom.xml index fcbd7f29f3e..83eff87c270 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/pom.xml b/pom.xml index 00b7f403c1c..6e01a25aab4 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.13-SNAPSHOT + 0.4.12.1 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index c12f4bd0d84..9438e5f867b 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/server/pom.xml b/server/pom.xml index 194991b7c56..dfd361a00a1 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 diff --git a/services/pom.xml b/services/pom.xml index 0c2c9d7dc66..d5739e16eb3 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.13-SNAPSHOT + 0.4.12.1 com.metamx druid - 0.4.13-SNAPSHOT + 0.4.12.1 From f80e85517cffe4e8924f1a1c1587b77b27795843 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 17 May 2013 11:14:16 -0700 Subject: [PATCH 03/12] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 8d1fabd80dd..61fe7cccb18 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 61e683d79e6..15acfaa632c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 08eb17f7957..dd2f62dc476 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 886f041eac2..8d70d7f649f 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 26b398984d9..c6c9ea6bcab 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index 83eff87c270..2ab44e7e470 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/pom.xml b/pom.xml index 6e01a25aab4..91119c68203 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.1 + 0.4.12.2-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 9438e5f867b..f1da629ce65 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index dfd361a00a1..eecfd886d2f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index d5739e16eb3..5c819bdc219 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.1 + 0.4.12.2-SNAPSHOT com.metamx druid - 0.4.12.1 + 0.4.12.2-SNAPSHOT From 31f5cb5c8f5fce499de2657c93994759c4ff699f Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 14 May 2013 14:39:29 -0700 Subject: [PATCH 04/12] DbConnectorConfig: Serialization is hard --- .../java/com/metamx/druid/db/DbConnector.java | 2 +- .../metamx/druid/db/DbConnectorConfig.java | 2 +- .../indexer/HadoopDruidIndexerConfigTest.java | 55 ++++++++++++++++--- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/com/metamx/druid/db/DbConnector.java b/common/src/main/java/com/metamx/druid/db/DbConnector.java index 1e73b731353..cb202b1a8f2 100644 --- a/common/src/main/java/com/metamx/druid/db/DbConnector.java +++ b/common/src/main/java/com/metamx/druid/db/DbConnector.java @@ -181,7 +181,7 @@ public class DbConnector dataSource.setPassword(config.getDatabasePassword()); dataSource.setUrl(config.getDatabaseConnectURI()); - if (config.isValidationQuery()) { + if (config.useValidationQuery()) { dataSource.setValidationQuery(config.getValidationQuery()); dataSource.setTestOnBorrow(true); } diff --git a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java index fb7d1a99916..e17302a3183 100644 --- a/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java +++ b/common/src/main/java/com/metamx/druid/db/DbConnectorConfig.java @@ -44,7 +44,7 @@ public abstract class DbConnectorConfig @JsonProperty("useValidationQuery") @Config("druid.database.validation") - public boolean isValidationQuery() { + public boolean useValidationQuery() { return false; } diff --git a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java index 5fdff8ce8b8..5745c3ab3e9 100644 --- a/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexer/src/test/java/com/metamx/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.metamx.druid.indexer.granularity.UniformGranularitySpec; import com.metamx.druid.indexer.partitions.PartitionsSpec; +import com.metamx.druid.indexer.updater.DbUpdaterJobSpec; import com.metamx.druid.jackson.DefaultObjectMapper; import org.joda.time.Interval; @@ -39,8 +40,8 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( - "{" + cfg = jsonReadWriteRead( + "{" + " \"granularitySpec\":{" + " \"type\":\"uniform\"," + " \"gran\":\"hour\"," @@ -74,7 +75,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"segmentGranularity\":\"day\"," + "\"intervals\":[\"2012-02-01/P1D\"]" @@ -137,7 +138,7 @@ public class HadoopDruidIndexerConfigTest public void testInvalidGranularityCombination() { boolean thrown = false; try { - final HadoopDruidIndexerConfig cfg = jsonMapper.readValue( + final HadoopDruidIndexerConfig cfg = jsonReadWriteRead( "{" + "\"segmentGranularity\":\"day\"," + "\"intervals\":[\"2012-02-01/P1D\"]," @@ -161,7 +162,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{}", HadoopDruidIndexerConfig.class ); @@ -183,7 +184,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"partitionsSpec\":{" + " \"targetPartitionSize\":100" @@ -221,7 +222,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"partitionsSpec\":{" + " \"targetPartitionSize\":100," @@ -260,7 +261,7 @@ public class HadoopDruidIndexerConfigTest final HadoopDruidIndexerConfig cfg; try { - cfg = jsonMapper.readValue( + cfg = jsonReadWriteRead( "{" + "\"targetPartitionSize\":100," + "\"partitionDimension\":\"foo\"" @@ -296,7 +297,7 @@ public class HadoopDruidIndexerConfigTest public void testInvalidPartitionsCombination() { boolean thrown = false; try { - final HadoopDruidIndexerConfig cfg = jsonMapper.readValue( + final HadoopDruidIndexerConfig cfg = jsonReadWriteRead( "{" + "\"targetPartitionSize\":100," + "\"partitionsSpec\":{" @@ -311,4 +312,40 @@ public class HadoopDruidIndexerConfigTest Assert.assertTrue("Exception thrown", thrown); } + + @Test + public void testDbUpdaterJobSpec() throws Exception + { + final HadoopDruidIndexerConfig cfg; + + cfg = jsonReadWriteRead( + "{" + + "\"updaterJobSpec\":{\n" + + " \"type\" : \"db\",\n" + + " \"connectURI\" : \"jdbc:mysql://localhost/druid\",\n" + + " \"user\" : \"rofl\",\n" + + " \"password\" : \"p4ssw0rd\",\n" + + " \"segmentTable\" : \"segments\"\n" + + " }" + + "}", + HadoopDruidIndexerConfig.class + ); + + final DbUpdaterJobSpec spec = (DbUpdaterJobSpec) cfg.getUpdaterJobSpec(); + Assert.assertEquals("segments", spec.getSegmentTable()); + Assert.assertEquals("jdbc:mysql://localhost/druid", spec.getDatabaseConnectURI()); + Assert.assertEquals("rofl", spec.getDatabaseUser()); + Assert.assertEquals("p4ssw0rd", spec.getDatabasePassword()); + Assert.assertEquals(false, spec.useValidationQuery()); + } + + private T jsonReadWriteRead(String s, Class klass) + { + try { + return jsonMapper.readValue(jsonMapper.writeValueAsBytes(jsonMapper.readValue(s, klass)), klass); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } } From 7c687aeb73ea06e68fd50c6d6223660c1616f021 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 17 May 2013 12:06:16 -0700 Subject: [PATCH 05/12] [maven-release-plugin] prepare release druid-0.4.12.2 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 61fe7cccb18..9f53acedc0d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/common/pom.xml b/common/pom.xml index 15acfaa632c..d281011961f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/examples/pom.xml b/examples/pom.xml index dd2f62dc476..7c0e428cd5a 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/index-common/pom.xml b/index-common/pom.xml index 8d70d7f649f..c8d1fe8050a 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/indexer/pom.xml b/indexer/pom.xml index c6c9ea6bcab..adac3f37f82 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/merger/pom.xml b/merger/pom.xml index 2ab44e7e470..25cccd36e48 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/pom.xml b/pom.xml index 91119c68203..757ca5cef59 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.2-SNAPSHOT + 0.4.12.2 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index f1da629ce65..4ab5ee510a8 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/server/pom.xml b/server/pom.xml index eecfd886d2f..3041c7ec00b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 diff --git a/services/pom.xml b/services/pom.xml index 5c819bdc219..c6f2d6e5793 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.2-SNAPSHOT + 0.4.12.2 com.metamx druid - 0.4.12.2-SNAPSHOT + 0.4.12.2 From e76fd18a3640f8bab572fdae663208efca7b4a53 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 17 May 2013 12:06:24 -0700 Subject: [PATCH 06/12] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 9f53acedc0d..8c3b01c9833 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index d281011961f..4bfdfc16969 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 7c0e428cd5a..5417442752e 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index c8d1fe8050a..e571be25292 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index adac3f37f82..d5d1f449141 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index 25cccd36e48..3efeb3a56a6 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index 757ca5cef59..a3efa5bc6f8 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.2 + 0.4.12.3-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 4ab5ee510a8..fe7ad7c284c 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 3041c7ec00b..ddf240d539f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index c6f2d6e5793..2f2b1091fa1 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.2 + 0.4.12.3-SNAPSHOT com.metamx druid - 0.4.12.2 + 0.4.12.3-SNAPSHOT From 8af9598157446201010e403686bf5eb387eadc27 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:21:45 -0700 Subject: [PATCH 07/12] fix currSize bug and maxTime bugs --- .../com/metamx/druid/client/DruidServer.java | 28 +++++++++++-------- .../v1/IncrementalIndexStorageAdapter.java | 21 +++++++++----- .../v1/QueryableIndexStorageAdapter.java | 7 +++-- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/client/DruidServer.java b/client/src/main/java/com/metamx/druid/client/DruidServer.java index e3491dd82d8..874343e127e 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidServer.java +++ b/client/src/main/java/com/metamx/druid/client/DruidServer.java @@ -124,7 +124,7 @@ public class DruidServer implements Comparable return segments.get(segmentName); } - public DruidServer addDataSegment(String segmentName, DataSegment segment) + public DruidServer addDataSegment(String segmentId, DataSegment segment) { synchronized (lock) { String dataSourceName = segment.getDataSource(); @@ -138,10 +138,11 @@ public class DruidServer implements Comparable dataSources.put(dataSourceName, dataSource); } - dataSource.addSegment(segmentName, segment); - segments.put(segmentName, segment); - - currSize += segment.getSize(); + dataSource.addSegment(segmentId, segment); + if (!segments.containsKey(segmentId)) { + segments.put(segmentId, segment); + currSize += segment.getSize(); + } } return this; } @@ -156,13 +157,13 @@ public class DruidServer implements Comparable return this; } - public DruidServer removeDataSegment(String segmentName) + public DruidServer removeDataSegment(String segmentId) { synchronized (lock) { - DataSegment segment = segments.get(segmentName); + DataSegment segment = segments.get(segmentId); if (segment == null) { - log.warn("Asked to remove data segment that doesn't exist!? server[%s], segment[%s]", getName(), segmentName); + log.warn("Asked to remove data segment that doesn't exist!? server[%s], segment[%s]", getName(), segmentId); return this; } @@ -172,18 +173,21 @@ public class DruidServer implements Comparable log.warn( "Asked to remove data segment from dataSource[%s] that doesn't exist, but the segment[%s] exists!?!?!?! wtf? server[%s]", segment.getDataSource(), - segmentName, + segmentId, getName() ); return this; } - dataSource.removePartition(segmentName); - segments.remove(segmentName); + dataSource.removePartition(segmentId); + if (segments.containsKey(segmentId)) { + segments.remove(segmentId); + currSize -= segment.getSize(); + } + if (dataSource.isEmpty()) { dataSources.remove(dataSource.getName()); } - currSize -= segment.getSize(); } return this; diff --git a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java index 479e93bf99e..9e5694ef4e4 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java @@ -115,11 +115,14 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter public Iterable makeCursors(final Filter filter, final Interval interval, final QueryGranularity gran) { Interval actualIntervalTmp = interval; - Interval dataInterval = getInterval(); - if (!actualIntervalTmp.overlaps(dataInterval)) { + final Interval indexInterval = getInterval(); + + if (!actualIntervalTmp.overlaps(indexInterval)) { return ImmutableList.of(); } + final Interval dataInterval = new Interval(getMinTime().getMillis(), gran.next(getMaxTime().getMillis())); + if (actualIntervalTmp.getStart().isBefore(dataInterval.getStart())) { actualIntervalTmp = actualIntervalTmp.withStart(dataInterval.getStart()); } @@ -367,7 +370,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter final String columnName = column.toLowerCase(); final Integer metricIndexInt = index.getMetricIndex(columnName); - if(metricIndexInt != null) { + if (metricIndexInt != null) { final int metricIndex = metricIndexInt; final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(index.getMetricType(columnName)); @@ -390,7 +393,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter final Integer dimensionIndexInt = index.getDimensionIndex(columnName); - if(dimensionIndexInt != null) { + if (dimensionIndexInt != null) { final int dimensionIndex = dimensionIndexInt; return new ObjectColumnSelector() { @@ -404,10 +407,14 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter public String get() { final String[] dimVals = currEntry.getKey().getDims()[dimensionIndex]; - if(dimVals.length == 1) return dimVals[0]; - if(dimVals.length == 0) return null; + if (dimVals.length == 1) { + return dimVals[0]; + } + if (dimVals.length == 0) { + return null; + } throw new UnsupportedOperationException( - "makeObjectColumnSelector does not support multivalued columns" + "makeObjectColumnSelector does not support multivalued columns" ); } }; diff --git a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java index d80e44b4b9b..2e3dcaa3b16 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java @@ -136,11 +136,14 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter public Iterable makeCursors(Filter filter, Interval interval, QueryGranularity gran) { Interval actualInterval = interval; - final Interval dataInterval = getInterval(); - if (!actualInterval.overlaps(dataInterval)) { + final Interval indexInterval = getInterval(); + + if (!actualInterval.overlaps(indexInterval)) { return ImmutableList.of(); } + final Interval dataInterval = new Interval(getMinTime().getMillis(), gran.next(getMaxTime().getMillis())); + if (actualInterval.getStart().isBefore(dataInterval.getStart())) { actualInterval = actualInterval.withStart(dataInterval.getStart()); } From a66425fe2f398bc9319b94b8938328f571f1c632 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:32:33 -0700 Subject: [PATCH 08/12] fix according to CR --- .../com/metamx/druid/client/DruidServer.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/client/DruidServer.java b/client/src/main/java/com/metamx/druid/client/DruidServer.java index 874343e127e..6fbde3efa2d 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidServer.java +++ b/client/src/main/java/com/metamx/druid/client/DruidServer.java @@ -139,10 +139,16 @@ public class DruidServer implements Comparable } dataSource.addSegment(segmentId, segment); - if (!segments.containsKey(segmentId)) { - segments.put(segmentId, segment); - currSize += segment.getSize(); + + DataSegment shouldNotExist = segments.get(segmentId); + + if (shouldNotExist != null) { + log.warn("Asked to add data segment that already exists!? server[%s], segment[%s]", getName(), segmentId); + return this; } + + segments.put(segmentId, segment); + currSize += segment.getSize(); } return this; } @@ -180,10 +186,9 @@ public class DruidServer implements Comparable } dataSource.removePartition(segmentId); - if (segments.containsKey(segmentId)) { - segments.remove(segmentId); - currSize -= segment.getSize(); - } + + segments.remove(segmentId); + currSize -= segment.getSize(); if (dataSource.isEmpty()) { dataSources.remove(dataSource.getName()); From d197eafe1ee239eecdc2d7504682fefbbdc47974 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:37:18 -0700 Subject: [PATCH 09/12] [maven-release-plugin] prepare release druid-0.4.12.3 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 8c3b01c9833..bd7112ec717 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/common/pom.xml b/common/pom.xml index 4bfdfc16969..80ed85cf8b7 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/examples/pom.xml b/examples/pom.xml index 5417442752e..192b8073dda 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/index-common/pom.xml b/index-common/pom.xml index e571be25292..0d88f0ce480 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/indexer/pom.xml b/indexer/pom.xml index d5d1f449141..6b9b4925098 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/merger/pom.xml b/merger/pom.xml index 3efeb3a56a6..adf2786fb58 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/pom.xml b/pom.xml index a3efa5bc6f8..c024b6d7041 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.3-SNAPSHOT + 0.4.12.3 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index fe7ad7c284c..95151cc34b6 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/server/pom.xml b/server/pom.xml index ddf240d539f..3d506856d77 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 diff --git a/services/pom.xml b/services/pom.xml index 2f2b1091fa1..2709cb9c6cf 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.3-SNAPSHOT + 0.4.12.3 com.metamx druid - 0.4.12.3-SNAPSHOT + 0.4.12.3 From 0e2fe69130343167c7867f349e1a1d5e50793da9 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:37:27 -0700 Subject: [PATCH 10/12] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index bd7112ec717..dc56af4e36d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 80ed85cf8b7..66b393ff07e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 192b8073dda..7c1cb6880c1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 0d88f0ce480..007301ed67e 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 6b9b4925098..b5a05b79876 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index adf2786fb58..8a478896658 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/pom.xml b/pom.xml index c024b6d7041..523dd7b9678 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.3 + 0.4.12.4-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 95151cc34b6..e6976bb9e4e 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 3d506856d77..2c826f59bc6 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 2709cb9c6cf..4b2ae608652 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.3 + 0.4.12.4-SNAPSHOT com.metamx druid - 0.4.12.3 + 0.4.12.4-SNAPSHOT From dd11c757ea8d6933d2580d5528da2e2e07ff83c9 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:44:47 -0700 Subject: [PATCH 11/12] [maven-release-plugin] prepare release druid-0.4.12.4 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index dc56af4e36d..3f22de2cb87 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/common/pom.xml b/common/pom.xml index 66b393ff07e..2d5e6238a44 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/examples/pom.xml b/examples/pom.xml index 7c1cb6880c1..60d00c4f389 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/index-common/pom.xml b/index-common/pom.xml index 007301ed67e..60d13d8d53c 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/indexer/pom.xml b/indexer/pom.xml index b5a05b79876..7f01030978b 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/merger/pom.xml b/merger/pom.xml index 8a478896658..ea0fbba7775 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/pom.xml b/pom.xml index 523dd7b9678..636ae0eecc4 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.4-SNAPSHOT + 0.4.12.4 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index e6976bb9e4e..8cbacb99df7 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/server/pom.xml b/server/pom.xml index 2c826f59bc6..2fbcd0d5e28 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 diff --git a/services/pom.xml b/services/pom.xml index 4b2ae608652..0e3466d108a 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.4-SNAPSHOT + 0.4.12.4 com.metamx druid - 0.4.12.4-SNAPSHOT + 0.4.12.4 From d8265376728888a16f24f85b23536e5b19fc6615 Mon Sep 17 00:00:00 2001 From: fjy Date: Sat, 18 May 2013 13:44:54 -0700 Subject: [PATCH 12/12] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 3f22de2cb87..ade5dfee23d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 2d5e6238a44..be1b6c70433 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 60d00c4f389..369844f985c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index 60d13d8d53c..bc9bdae8032 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index 7f01030978b..f86da5c684f 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index ea0fbba7775..bf60cea247b 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/pom.xml b/pom.xml index 636ae0eecc4..3e55b83fbfc 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.4.12.4 + 0.4.12.5-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 8cbacb99df7..dd9c3225176 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 2fbcd0d5e28..6fc5e0cb39c 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 0e3466d108a..ca98d8794e1 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.4.12.4 + 0.4.12.5-SNAPSHOT com.metamx druid - 0.4.12.4 + 0.4.12.5-SNAPSHOT