From 083a012aa2cc34fce941c6bc58bda797b3cb597e Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 24 Nov 2014 14:54:11 -0800 Subject: [PATCH 01/14] Commonalize the cache config and change default to disable cache --- examples/config/_common/common.runtime.properties | 4 ++++ examples/config/broker/runtime.properties | 5 ++++- examples/config/coordinator/runtime.properties | 6 ++++-- examples/config/historical/runtime.properties | 5 +++-- examples/config/overlord/runtime.properties | 1 + examples/config/realtime/runtime.properties | 6 ++++-- server/src/main/java/io/druid/client/cache/CacheConfig.java | 4 ++-- services/src/main/java/io/druid/cli/CliBroker.java | 2 +- services/src/main/java/io/druid/cli/CliHistorical.java | 2 +- 9 files changed, 24 insertions(+), 11 deletions(-) diff --git a/examples/config/_common/common.runtime.properties b/examples/config/_common/common.runtime.properties index c73d92bec8c..348532cd582 100644 --- a/examples/config/_common/common.runtime.properties +++ b/examples/config/_common/common.runtime.properties @@ -14,6 +14,10 @@ druid.metadata.storage.connector.password=diurd druid.storage.type=local druid.storage.storage.storageDirectory=/tmp/druid/localStorage +# Cache (we use a simple 10mb heap-based local cache on the broker) +druid.cache.type=local +druid.cache.sizeInBytes=10000000 + # Indexing service discovery druid.selectors.indexing.serviceName=overlord diff --git a/examples/config/broker/runtime.properties b/examples/config/broker/runtime.properties index 23d1170343b..330918e5f26 100644 --- a/examples/config/broker/runtime.properties +++ b/examples/config/broker/runtime.properties @@ -1,6 +1,9 @@ druid.host=localhost -druid.service=broker druid.port=8080 +druid.service=broker + +druid.broker.cache.useCache=true +druid.broker.cache.populateCache=true # Bump these up only for faster nested groupBy druid.processing.buffer.sizeBytes=100000000 diff --git a/examples/config/coordinator/runtime.properties b/examples/config/coordinator/runtime.properties index c9f16857af4..7d0e82c97c9 100644 --- a/examples/config/coordinator/runtime.properties +++ b/examples/config/coordinator/runtime.properties @@ -1,5 +1,7 @@ druid.host=localhost -druid.service=coordinator druid.port=8082 +druid.service=coordinator -druid.coordinator.startDelay=PT70s \ No newline at end of file +# The coordinator begins assignment operations after the start delay. +# We override the default here to start things up faster for examples. +druid.coordinator.startDelay=PT70s diff --git a/examples/config/historical/runtime.properties b/examples/config/historical/runtime.properties index 332aa0c95b0..cbc5c138ee2 100644 --- a/examples/config/historical/runtime.properties +++ b/examples/config/historical/runtime.properties @@ -1,8 +1,9 @@ druid.host=localhost -druid.service=historical druid.port=8081 +druid.service=historical -# Change these to make Druid faster +# We can only 1 scan segment in parallel with these configs. +# Our intermediate buffer is also very small so longer topNs will be slow. druid.processing.buffer.sizeBytes=100000000 druid.processing.numThreads=1 diff --git a/examples/config/overlord/runtime.properties b/examples/config/overlord/runtime.properties index 3eebd15a9a9..6c5653fb5bc 100644 --- a/examples/config/overlord/runtime.properties +++ b/examples/config/overlord/runtime.properties @@ -2,6 +2,7 @@ druid.host=localhost druid.port=8080 druid.service=overlord +# Run the overlord in local mode with a single peon to execute tasks druid.indexer.queue.startDelay=PT0M druid.indexer.runner.javaOpts="-server -Xmx256m" druid.indexer.fork.property.druid.processing.numThreads=1 diff --git a/examples/config/realtime/runtime.properties b/examples/config/realtime/runtime.properties index 8a69a193277..0565853edd7 100644 --- a/examples/config/realtime/runtime.properties +++ b/examples/config/realtime/runtime.properties @@ -1,10 +1,12 @@ druid.host=localhost -druid.service=realtime druid.port=8083 +druid.service=realtime -# Change this config to metadata to hand off to the rest of the Druid cluster +# Change this config to 'metadata' to hand off to the rest of the Druid cluster druid.publish.type=noop +# We can only 1 scan segment in parallel with these configs. +# Our intermediate buffer is also very small so longer topNs will be slow. druid.processing.buffer.sizeBytes=100000000 druid.processing.numThreads=1 diff --git a/server/src/main/java/io/druid/client/cache/CacheConfig.java b/server/src/main/java/io/druid/client/cache/CacheConfig.java index 25c397b7a83..2e066e6703d 100644 --- a/server/src/main/java/io/druid/client/cache/CacheConfig.java +++ b/server/src/main/java/io/druid/client/cache/CacheConfig.java @@ -31,10 +31,10 @@ public class CacheConfig public static final String POPULATE_CACHE = "populateCache"; @JsonProperty - private boolean useCache = true; + private boolean useCache = false; @JsonProperty - private boolean populateCache = true; + private boolean populateCache = false; @JsonProperty private List unCacheable = Arrays.asList(Query.GROUP_BY, Query.SELECT); diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index 877a356103e..9050bc827aa 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -90,7 +90,7 @@ public class CliBroker extends ServerRunnable binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); + JsonConfigProvider.bind(binder, "druid.cache", CacheProvider.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index c34b91d2188..79c28b24c54 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -88,7 +88,7 @@ public class CliHistorical extends ServerRunnable LifecycleModule.register(binder, ZkCoordinator.class); binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); + JsonConfigProvider.bind(binder, "druid.cache", CacheProvider.class); JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); MetricsModule.register(binder, CacheMonitor.class); } From 7cd689be75c1b106642d13b64a68d3e0ec254efb Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 2 Dec 2014 11:01:12 -0800 Subject: [PATCH 02/14] Fix Hadoop CLI jobs * Change "schema" --> "spec" for cli hadoop to keep up with internal hadoop * Added check for HadoopDruidIndexerConfig deserialization from Map to see if it is trying to get a HadoopDruidIndexerConfig or a HadoopIngestionSpec --- .../indexer/HadoopDruidIndexerConfig.java | 28 +++++++++++++------ .../java/io/druid/indexer/HadoopIOConfig.java | 2 ++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java index e33a9c30fb5..e54803cee72 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java @@ -114,12 +114,21 @@ public class HadoopDruidIndexerConfig public static HadoopDruidIndexerConfig fromMap(Map argSpec) { - return new HadoopDruidIndexerConfig( - HadoopDruidIndexerConfig.jsonMapper.convertValue( - argSpec, - HadoopIngestionSpec.class - ) + // Eventually PathSpec needs to get rid of its Hadoop dependency, then maybe this can be ingested directly without + // the Map<> intermediary + + if(argSpec.containsKey("spec")){ + return HadoopDruidIndexerConfig.jsonMapper.convertValue( + argSpec, + HadoopDruidIndexerConfig.class ); + } + return new HadoopDruidIndexerConfig( + HadoopDruidIndexerConfig.jsonMapper.convertValue( + argSpec, + HadoopIngestionSpec.class + ) + ); } @SuppressWarnings("unchecked") @@ -142,12 +151,13 @@ public class HadoopDruidIndexerConfig @SuppressWarnings("unchecked") public static HadoopDruidIndexerConfig fromString(String str) { + // This is a map to try and prevent dependency screwbally-ness try { return fromMap( (Map) HadoopDruidIndexerConfig.jsonMapper.readValue( str, new TypeReference>() - { - } + { + } ) ); } @@ -171,7 +181,7 @@ public class HadoopDruidIndexerConfig @JsonCreator public HadoopDruidIndexerConfig( - final @JsonProperty("schema") HadoopIngestionSpec schema + final @JsonProperty("spec") HadoopIngestionSpec schema ) { this.schema = schema; @@ -202,7 +212,7 @@ public class HadoopDruidIndexerConfig this.rollupGran = schema.getDataSchema().getGranularitySpec().getQueryGranularity(); } - @JsonProperty + @JsonProperty(value="spec") public HadoopIngestionSpec getSchema() { return schema; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java index 3aca5b6136b..fdd858e2b27 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java @@ -19,6 +19,7 @@ package io.druid.indexer; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import io.druid.indexer.updater.MetadataStorageUpdaterJobSpec; @@ -35,6 +36,7 @@ public class HadoopIOConfig implements IOConfig private final MetadataStorageUpdaterJobSpec metadataUpdateSpec; private final String segmentOutputPath; + @JsonCreator public HadoopIOConfig( final @JsonProperty("inputSpec") Map pathSpec, final @JsonProperty("metadataUpdateSpec") MetadataStorageUpdaterJobSpec metadataUpdateSpec, From a79389a9e5992b7107a2409e0f3f463265ffeadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 2 Dec 2014 16:55:32 -0800 Subject: [PATCH 03/14] update joda-time and fix min/max instant --- common/src/main/java/io/druid/common/utils/JodaUtils.java | 4 ++++ .../io/druid/timeline/VersionedIntervalTimelineTest.java | 3 ++- .../main/java/io/druid/indexing/overlord/TaskLockbox.java | 5 +++-- pom.xml | 2 +- .../src/main/java/io/druid/granularity/AllGranularity.java | 5 +++-- .../java/io/druid/query/timeboundary/TimeBoundaryQuery.java | 5 +++-- processing/src/main/java/io/druid/segment/IndexMaker.java | 4 ++-- processing/src/main/java/io/druid/segment/IndexMerger.java | 4 ++-- .../indexing/granularity/ArbitraryGranularitySpec.java | 3 ++- .../realtime/plumber/MessageTimeRejectionPolicyFactory.java | 3 ++- .../segment/realtime/plumber/RealtimePlumberSchoolTest.java | 2 +- 11 files changed, 25 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/io/druid/common/utils/JodaUtils.java b/common/src/main/java/io/druid/common/utils/JodaUtils.java index db6da04e875..a3e1b9cee26 100644 --- a/common/src/main/java/io/druid/common/utils/JodaUtils.java +++ b/common/src/main/java/io/druid/common/utils/JodaUtils.java @@ -36,6 +36,10 @@ import java.util.TreeSet; */ public class JodaUtils { + // joda limits years to [-292275054,292278993] that should be reasonable + public static final long MAX_INSTANT = new DateTime("292278993").getMillis(); + public static final long MIN_INSTANT = new DateTime("-292275054").getMillis(); + public static ArrayList condenseIntervals(Iterable intervals) { ArrayList retVal = Lists.newArrayList(); diff --git a/common/src/test/java/io/druid/timeline/VersionedIntervalTimelineTest.java b/common/src/test/java/io/druid/timeline/VersionedIntervalTimelineTest.java index 54a358223a6..e5611f6c332 100644 --- a/common/src/test/java/io/druid/timeline/VersionedIntervalTimelineTest.java +++ b/common/src/test/java/io/druid/timeline/VersionedIntervalTimelineTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.metamx.common.Pair; +import io.druid.common.utils.JodaUtils; import io.druid.timeline.partition.ImmutablePartitionHolder; import io.druid.timeline.partition.IntegerPartitionChunk; import io.druid.timeline.partition.PartitionChunk; @@ -1294,7 +1295,7 @@ public class VersionedIntervalTimelineTest createExpected("2011-01-05/2011-01-10", "2", 2), createExpected("2011-01-10/2011-01-15", "3", 3) ), - timeline.lookup(new Interval(new DateTime(0), new DateTime(Long.MAX_VALUE))) + timeline.lookup(new Interval(new DateTime(0), new DateTime(JodaUtils.MAX_INSTANT))) ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java index 28506499f4c..90962e55705 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java @@ -36,6 +36,7 @@ import com.metamx.common.Pair; import com.metamx.common.guava.Comparators; import com.metamx.common.guava.FunctionalIterable; import com.metamx.emitter.EmittingLogger; +import io.druid.common.utils.JodaUtils; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.task.Task; import org.joda.time.DateTime; @@ -462,11 +463,11 @@ public class TaskLockbox final NavigableSet dsLockbox = dsRunning.navigableKeySet(); final Iterable searchIntervals = Iterables.concat( // Single interval that starts at or before ours - Collections.singletonList(dsLockbox.floor(new Interval(interval.getStart(), new DateTime(Long.MAX_VALUE)))), + Collections.singletonList(dsLockbox.floor(new Interval(interval.getStart(), new DateTime(JodaUtils.MAX_INSTANT)))), // All intervals that start somewhere between our start instant (exclusive) and end instant (exclusive) dsLockbox.subSet( - new Interval(interval.getStart(), new DateTime(Long.MAX_VALUE)), + new Interval(interval.getStart(), new DateTime(JodaUtils.MAX_INSTANT)), false, new Interval(interval.getEnd(), interval.getEnd()), false diff --git a/pom.xml b/pom.xml index 35fa76c921b..7562dee7406 100644 --- a/pom.xml +++ b/pom.xml @@ -348,7 +348,7 @@ joda-time joda-time - 2.1 + 2.6 com.google.code.findbugs diff --git a/processing/src/main/java/io/druid/granularity/AllGranularity.java b/processing/src/main/java/io/druid/granularity/AllGranularity.java index eebe4c43ab1..165c0a34134 100644 --- a/processing/src/main/java/io/druid/granularity/AllGranularity.java +++ b/processing/src/main/java/io/druid/granularity/AllGranularity.java @@ -20,19 +20,20 @@ package io.druid.granularity; import com.google.common.collect.ImmutableList; +import io.druid.common.utils.JodaUtils; public final class AllGranularity extends BaseQueryGranularity { @Override public long next(long offset) { - return Long.MAX_VALUE; + return JodaUtils.MAX_INSTANT; } @Override public long truncate(long offset) { - return Long.MIN_VALUE; + return JodaUtils.MIN_INSTANT; } @Override diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java index ec2f32174fc..03adf942695 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.common.utils.JodaUtils; import io.druid.query.BaseQuery; import io.druid.query.DataSource; import io.druid.query.Query; @@ -155,8 +156,8 @@ public class TimeBoundaryQuery extends BaseQuery return Lists.newArrayList(); } - DateTime min = new DateTime(Long.MAX_VALUE); - DateTime max = new DateTime(Long.MIN_VALUE); + DateTime min = new DateTime(JodaUtils.MAX_INSTANT); + DateTime max = new DateTime(JodaUtils.MIN_INSTANT); for (Result result : results) { TimeBoundaryResultValue val = result.getValue(); diff --git a/processing/src/main/java/io/druid/segment/IndexMaker.java b/processing/src/main/java/io/druid/segment/IndexMaker.java index ac9708723d6..4f6bf51605b 100644 --- a/processing/src/main/java/io/druid/segment/IndexMaker.java +++ b/processing/src/main/java/io/druid/segment/IndexMaker.java @@ -1363,8 +1363,8 @@ public class IndexMaker cols.writeToChannel(writer); dims.writeToChannel(writer); - DateTime minTime = new DateTime(Long.MAX_VALUE); - DateTime maxTime = new DateTime(0l); + DateTime minTime = new DateTime(JodaUtils.MAX_INSTANT); + DateTime maxTime = new DateTime(JodaUtils.MIN_INSTANT); for (IndexableAdapter index : adapters) { minTime = JodaUtils.minDateTime(minTime, index.getDataInterval().getStart()); diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 3021ff83934..fdd2de05d02 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -457,8 +457,8 @@ public class IndexMerger GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.stringStrategy).writeToChannel(channel); GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.stringStrategy).writeToChannel(channel); - DateTime minTime = new DateTime(Long.MAX_VALUE); - DateTime maxTime = new DateTime(0l); + DateTime minTime = new DateTime(JodaUtils.MAX_INSTANT); + DateTime maxTime = new DateTime(JodaUtils.MIN_INSTANT); for (IndexableAdapter index : indexes) { minTime = JodaUtils.minDateTime(minTime, index.getDataInterval().getStart()); diff --git a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java index 627667a5941..36a30eea391 100644 --- a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java +++ b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java @@ -28,6 +28,7 @@ import com.google.common.collect.PeekingIterator; import com.google.common.collect.Sets; import com.metamx.common.Granularity; import com.metamx.common.guava.Comparators; +import io.druid.common.utils.JodaUtils; import io.druid.granularity.QueryGranularity; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -90,7 +91,7 @@ public class ArbitraryGranularitySpec implements GranularitySpec public Optional bucketInterval(DateTime dt) { // First interval with start time ≤ dt - final Interval interval = intervals.floor(new Interval(dt, new DateTime(Long.MAX_VALUE))); + final Interval interval = intervals.floor(new Interval(dt, new DateTime(JodaUtils.MAX_INSTANT))); if (interval != null && interval.contains(dt)) { return Optional.of(interval); diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/MessageTimeRejectionPolicyFactory.java b/server/src/main/java/io/druid/segment/realtime/plumber/MessageTimeRejectionPolicyFactory.java index 7b6c5868017..2064ff11956 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/MessageTimeRejectionPolicyFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/MessageTimeRejectionPolicyFactory.java @@ -19,6 +19,7 @@ package io.druid.segment.realtime.plumber; +import io.druid.common.utils.JodaUtils; import org.joda.time.DateTime; import org.joda.time.Period; @@ -31,7 +32,7 @@ public class MessageTimeRejectionPolicyFactory implements RejectionPolicyFactory return new RejectionPolicy() { - private volatile long maxTimestamp = Long.MIN_VALUE; + private volatile long maxTimestamp = JodaUtils.MIN_INSTANT; @Override public DateTime getCurrMaxTime() diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index 964325dd201..625c2813744 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -193,7 +193,7 @@ public class RealtimePlumberSchoolTest public void testPersist() throws Exception { final MutableBoolean committed = new MutableBoolean(false); - plumber.getSinks().put(0L, new Sink(new Interval(0, TimeUnit.HOURS.toMillis(1)),schema, tuningConfig, DateTime.now().toString())); + plumber.getSinks().put(0L, new Sink(new Interval(0, TimeUnit.HOURS.toMillis(1)),schema, tuningConfig, new DateTime("2014-12-01T12:34:56.789").toString())); plumber.startJob(); final InputRow row = EasyMock.createNiceMock(InputRow.class); EasyMock.expect(row.getTimestampFromEpoch()).andReturn(0L); From b59bfaa210c82228c3e76491f834cd0a9ef3d0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 2 Dec 2014 17:51:55 -0800 Subject: [PATCH 04/14] update jackson to 2.4.4 --- .../JacksonifiedConnectionFactory.java | 19 ++++++++++++++ pom.xml | 26 ++++++------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java index 28f1d0d14d1..2e9f18cfd92 100644 --- a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java +++ b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java @@ -20,11 +20,16 @@ package io.druid.firehose.rabbitmq; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Maps; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.client.LongString; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; import java.util.Map; /** @@ -154,6 +159,20 @@ public class JacksonifiedConnectionFactory extends ConnectionFactory return uri; } + // we are only overriding this to help Jackson not be confused about the two setURI methods + @JsonIgnore + @Override + public void setUri(URI uri) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException + { + super.setUri(uri); + } + + @Override + public void setUri(String uriString) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException + { + super.setUri(uriString); + } + @Override @JsonProperty public int getRequestedChannelMax() diff --git a/pom.xml b/pom.xml index 35fa76c921b..d681cf460e8 100644 --- a/pom.xml +++ b/pom.xml @@ -223,52 +223,42 @@ com.fasterxml.jackson.core jackson-annotations - 2.2.3 + 2.4.4 com.fasterxml.jackson.core jackson-core - 2.2.3 + 2.4.4 com.fasterxml.jackson.core jackson-databind - 2.2.3 + 2.4.4 com.fasterxml.jackson.datatype jackson-datatype-guava - 2.2.3 + 2.4.4 com.fasterxml.jackson.datatype jackson-datatype-joda - 2.2.3 + 2.4.4 com.fasterxml.jackson.dataformat jackson-dataformat-smile - 2.2.3 + 2.4.4 com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - 2.2.3 + 2.4.4 com.fasterxml.jackson.jaxrs jackson-jaxrs-smile-provider - 2.2.3 - - - org.codehaus.jackson - jackson-core-asl - 1.9.13 - - - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 + 2.4.4 org.hibernate From 18f50097a942177fe7341d8bde0cf1cc5a96b596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 2 Dec 2014 23:49:34 -0800 Subject: [PATCH 05/14] upgrade LZ4 to operate directly on ByteBuffers --- pom.xml | 2 +- .../data/CompressedObjectStrategy.java | 37 ++++--------------- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 35fa76c921b..1ba03e22afa 100644 --- a/pom.xml +++ b/pom.xml @@ -393,7 +393,7 @@ net.jpountz.lz4 lz4 - 1.2.0 + 1.3.0 com.google.protobuf diff --git a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java index feea7d8abbf..b646fd800c2 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java @@ -19,9 +19,7 @@ package io.druid.segment.data; -import com.google.common.base.Throwables; import com.google.common.collect.Maps; -import com.metamx.common.guava.CloseQuietly; import com.metamx.common.logger.Logger; import com.ning.compress.lzf.ChunkEncoder; import com.ning.compress.lzf.LZFChunk; @@ -34,7 +32,6 @@ import net.jpountz.lz4.LZ4SafeDecompressor; import java.io.IOException; import java.nio.Buffer; -import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Map; @@ -233,38 +230,18 @@ public class CompressedObjectStrategy implements ObjectStrateg @Override public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) { - final byte[] bytes = new byte[numBytes]; - in.get(bytes); - - try (final ResourceHolder outputBytesHolder = CompressedPools.getOutputBytes()) { - final byte[] outputBytes = outputBytesHolder.get(); - // Since decompressed size is NOT known, must use lz4Safe - final int numDecompressedBytes = lz4Safe.decompress(bytes, outputBytes); - out.put(outputBytes, 0, numDecompressedBytes); - out.flip(); - } - catch (IOException e) { - log.error(e, "IOException thrown while closing ChunkEncoder."); - } + // Since decompressed size is NOT known, must use lz4Safe + // lz4Safe.decompress does not modify buffer positions + final int numDecompressedBytes = lz4Safe.decompress(in, in.position(), numBytes, out, out.position(), out.remaining()); + out.limit(out.position() + numDecompressedBytes); } @Override public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) { - final byte[] bytes = new byte[numBytes]; - in.get(bytes); - - // TODO: Upgrade this to ByteBuffer once https://github.com/jpountz/lz4-java/issues/9 is in mainline code for lz4-java - try (final ResourceHolder outputBytesHolder = CompressedPools.getOutputBytes()) { - final byte[] outputBytes = outputBytesHolder.get(); - lz4Fast.decompress(bytes, 0, outputBytes, 0, decompressedSize); - - out.put(outputBytes, 0, decompressedSize); - out.flip(); - } - catch (IOException e) { - log.error(e, "IOException thrown while closing ChunkEncoder."); - } + // lz4Fast.decompress does not modify buffer positions + lz4Fast.decompress(in, in.position(), out, out.position(), decompressedSize); + out.limit(out.position() + decompressedSize); } } From 5fece517fa7910be3a5fbf127c5b02ac7d861640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 3 Dec 2014 00:00:47 -0800 Subject: [PATCH 06/14] write byte data as is in smile --- processing/src/main/java/io/druid/jackson/JacksonModule.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/processing/src/main/java/io/druid/jackson/JacksonModule.java b/processing/src/main/java/io/druid/jackson/JacksonModule.java index 94e4e081deb..93dcfc888e2 100644 --- a/processing/src/main/java/io/druid/jackson/JacksonModule.java +++ b/processing/src/main/java/io/druid/jackson/JacksonModule.java @@ -21,6 +21,7 @@ package io.druid.jackson; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.smile.SmileFactory; +import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; @@ -49,6 +50,7 @@ public class JacksonModule implements Module public ObjectMapper smileMapper() { final SmileFactory smileFactory = new SmileFactory(); + smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); smileFactory.delegateToTextual(true); final ObjectMapper retVal = new DefaultObjectMapper(smileFactory); retVal.getFactory().setCodec(retVal); From 61f9289447fce3e6dd919c4ab1c73e5df554c150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 3 Dec 2014 09:44:19 -0800 Subject: [PATCH 07/14] use property for jackson version --- pom.xml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index d681cf460e8..b54349a4a7f 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ 2.7.0 9.2.5.v20141112 0.3.0 + 2.4.4 @@ -223,42 +224,42 @@ com.fasterxml.jackson.core jackson-annotations - 2.4.4 + ${jackson.version} com.fasterxml.jackson.core jackson-core - 2.4.4 + ${jackson.version} com.fasterxml.jackson.core jackson-databind - 2.4.4 + ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-guava - 2.4.4 + ${jackson.version} com.fasterxml.jackson.datatype jackson-datatype-joda - 2.4.4 + ${jackson.version} com.fasterxml.jackson.dataformat jackson-dataformat-smile - 2.4.4 + ${jackson.version} com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - 2.4.4 + ${jackson.version} com.fasterxml.jackson.jaxrs jackson-jaxrs-smile-provider - 2.4.4 + ${jackson.version} org.hibernate From 03ae70ae54bb4dac1a69bd0e692d759ab64f25ba Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 3 Dec 2014 10:43:24 -0800 Subject: [PATCH 08/14] SSL-capable ServiceEmitter. --- .../server/initialization/HttpEmitterModule.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java index f384287f726..4e7c083bee6 100644 --- a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java +++ b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java @@ -20,6 +20,7 @@ package io.druid.server.initialization; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; import com.google.common.base.Supplier; import com.google.inject.Binder; import com.google.inject.Module; @@ -37,6 +38,7 @@ import io.druid.guice.ManageLifecycle; import javax.annotation.Nullable; import javax.net.ssl.SSLContext; +import java.security.NoSuchAlgorithmException; /** */ @@ -47,8 +49,14 @@ public class HttpEmitterModule implements Module { JsonConfigProvider.bind(binder, "druid.emitter.http", HttpEmitterConfig.class); - // Fix the injection of this if we want to enable ssl emission of events. - binder.bind(SSLContext.class).toProvider(Providers.of(null)).in(LazySingleton.class); + final SSLContext context; + try { + context = SSLContext.getDefault(); + } catch (NoSuchAlgorithmException e) { + throw Throwables.propagate(e); + } + + binder.bind(SSLContext.class).toProvider(Providers.of(context)).in(LazySingleton.class); } @Provides From d388a8fe892af322c56d7696bcea6836525b8e58 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 3 Dec 2014 10:52:25 -0800 Subject: [PATCH 09/14] Replace google-http-client imports with real guava imports. --- .../java/io/druid/indexer/HadoopDruidIndexerConfig.java | 2 +- .../src/main/java/io/druid/indexer/JobHelper.java | 2 +- .../io/druid/indexer/HadoopDruidIndexerConfigTest.java | 2 +- .../druid/indexing/common/actions/TaskActionToolbox.java | 2 +- .../io/druid/indexing/common/task/HadoopIndexTask.java | 2 +- .../indexing/firehose/IngestSegmentFirehoseFactory.java | 2 +- .../io/druid/indexing/overlord/HeapMemoryTaskStorage.java | 2 +- .../main/java/io/druid/indexing/overlord/TaskLockbox.java | 2 +- .../main/java/io/druid/indexing/overlord/TaskQueue.java | 2 +- .../io/druid/indexing/overlord/ThreadPoolTaskRunner.java | 2 +- .../overlord/autoscaling/ec2/GalaxyEC2UserData.java | 2 +- .../overlord/autoscaling/ec2/StringEC2UserData.java | 2 +- .../druid/indexing/worker/executor/ExecutorLifecycle.java | 2 +- .../io/druid/indexing/worker/http/WorkerResource.java | 3 +-- .../java/io/druid/indexing/common/task/IndexTaskTest.java | 2 +- .../io/druid/indexing/overlord/RemoteTaskRunnerTest.java | 2 +- .../io/druid/indexing/overlord/TaskLifecycleTest.java | 8 ++------ .../java/io/druid/client/BatchServerInventoryView.java | 2 +- .../java/io/druid/client/SingleServerInventoryView.java | 2 +- .../druid/client/selector/CustomTierSelectorStrategy.java | 2 +- .../client/selector/CustomTierSelectorStrategyConfig.java | 2 +- .../java/io/druid/guice/http/JettyHttpClientModule.java | 2 +- .../druid/metadata/SQLMetadataStorageActionHandler.java | 3 +-- .../indexing/granularity/ArbitraryGranularitySpec.java | 2 +- .../java/io/druid/segment/realtime/FireDepartment.java | 6 ------ .../main/java/io/druid/segment/realtime/FireHydrant.java | 2 +- .../realtime/firehose/CombiningFirehoseFactory.java | 4 ++-- .../segment/realtime/firehose/LocalFirehoseFactory.java | 2 +- .../io/druid/server/coordinator/CostBalancerStrategy.java | 2 +- .../java/io/druid/server/coordinator/rules/LoadRule.java | 2 +- .../src/main/java/io/druid/server/http/TiersResource.java | 4 ++-- .../io/druid/server/router/CoordinatorRuleManager.java | 2 +- .../timeline/partition/HashBasedNumberedShardSpec.java | 2 +- .../server/coordination/CostBalancerStrategyTest.java | 4 ++-- .../io/druid/server/coordinator/rules/LoadRuleTest.java | 4 ++-- .../java/io/druid/server/initialization/JettyTest.java | 4 ++-- .../main/java/io/druid/cli/CliInternalHadoopIndexer.java | 6 +++--- services/src/main/java/io/druid/cli/PullDependencies.java | 2 +- .../cli/convert/DataSegmentPusherDefaultConverter.java | 2 +- .../io/druid/cli/convert/DatabasePropertiesConverter.java | 2 +- .../src/main/java/io/druid/cli/convert/PrefixRename.java | 2 +- .../java/io/druid/cli/validate/DruidJsonValidator.java | 2 +- 42 files changed, 49 insertions(+), 61 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java index e54803cee72..f7b168d0f3f 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Maps; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Optional; @@ -32,6 +31,7 @@ import com.google.common.base.Splitter; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Key; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index b20d31c44f1..b1ffabaa1f4 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -19,8 +19,8 @@ package io.druid.indexer; -import com.google.api.client.util.Sets; import com.google.common.base.Throwables; +import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.google.common.io.OutputSupplier; diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java index 2500954e944..e3f294b47a0 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -20,10 +20,10 @@ package io.druid.indexer; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Lists; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.metamx.common.Granularity; import io.druid.data.input.MapBasedInputRow; import io.druid.granularity.QueryGranularity; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java index 5b055b4ba90..a07c7b22821 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java @@ -19,7 +19,7 @@ package io.druid.indexing.common.actions; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.inject.Inject; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index 7087d0dbfcb..b797d772d83 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -23,12 +23,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.api.client.util.Lists; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.inject.Injector; import com.metamx.common.logger.Logger; import io.druid.common.utils.JodaUtils; diff --git a/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java b/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java index 37af09d0f57..7555a392631 100644 --- a/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java +++ b/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java @@ -22,9 +22,9 @@ package io.druid.indexing.firehose; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.repackaged.com.google.common.base.Throwables; import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/HeapMemoryTaskStorage.java b/indexing-service/src/main/java/io/druid/indexing/overlord/HeapMemoryTaskStorage.java index 55c86af6210..63c04d3769b 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/HeapMemoryTaskStorage.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/HeapMemoryTaskStorage.java @@ -19,12 +19,12 @@ package io.druid.indexing.overlord; -import com.google.api.client.util.Lists; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Ordering; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java index 90962e55705..bc59bb09a45 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java @@ -19,10 +19,10 @@ package io.druid.indexing.overlord; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Optional; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java index c309005616b..72225f243ee 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java @@ -19,7 +19,6 @@ package io.druid.indexing.overlord; -import com.google.api.client.util.Maps; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; @@ -27,6 +26,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java index 0b310aa401d..3961bd92a8e 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java @@ -19,7 +19,7 @@ package io.druid.indexing.overlord; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java index f3b238ca97a..1464324a053 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import io.druid.guice.annotations.Json; import org.apache.commons.codec.binary.Base64; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java index a017c0aad5b..cbb43441ba0 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java @@ -21,7 +21,7 @@ package io.druid.indexing.overlord.autoscaling.ec2; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.util.Charsets; +import com.google.common.base.Charsets; import org.apache.commons.codec.binary.Base64; public class StringEC2UserData implements EC2UserData diff --git a/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java b/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java index 17d889b5143..8ba0f64f632 100644 --- a/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java +++ b/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java @@ -20,8 +20,8 @@ package io.druid.indexing.worker.executor; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; diff --git a/indexing-service/src/main/java/io/druid/indexing/worker/http/WorkerResource.java b/indexing-service/src/main/java/io/druid/indexing/worker/http/WorkerResource.java index 38a5a52a1a9..1bf4936f8e0 100644 --- a/indexing-service/src/main/java/io/druid/indexing/worker/http/WorkerResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/worker/http/WorkerResource.java @@ -19,13 +19,12 @@ package io.druid.indexing.worker.http; -import com.google.api.client.util.Lists; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.io.ByteSource; -import com.google.common.io.InputSupplier; import com.google.inject.Inject; import com.metamx.common.logger.Logger; import io.druid.indexing.overlord.ForkingTaskRunner; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index 06b161951da..5a486e1cedb 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -19,7 +19,7 @@ package io.druid.indexing.common.task; -import com.google.api.client.util.Lists; +import com.google.common.collect.Lists; import com.metamx.common.Granularity; import io.druid.data.input.impl.CSVParseSpec; import io.druid.data.input.impl.DimensionsSpec; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index caecef4d5dd..a007c9f4cf2 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -20,9 +20,9 @@ package io.druid.indexing.overlord; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Throwables; import com.google.common.base.Function; import com.google.common.base.Joiner; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index ecf1bd3c4e1..4ae49dc5531 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -19,8 +19,8 @@ package io.druid.indexing.overlord; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; import com.google.common.base.Optional; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -42,8 +42,6 @@ import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.InputRowParser; -import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; -import io.druid.granularity.QueryGranularity; import io.druid.indexing.common.SegmentLoaderFactory; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskStatus; @@ -64,7 +62,7 @@ import io.druid.indexing.common.task.Task; import io.druid.indexing.common.task.TaskResource; import io.druid.indexing.overlord.config.TaskQueueConfig; import io.druid.jackson.DefaultObjectMapper; -import io.druid.metadata.SQLMetadataConnector; +import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.segment.indexing.DataSchema; @@ -88,8 +86,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.skife.jdbi.v2.DBI; -import org.skife.jdbi.v2.Handle; import java.io.File; import java.io.IOException; diff --git a/server/src/main/java/io/druid/client/BatchServerInventoryView.java b/server/src/main/java/io/druid/client/BatchServerInventoryView.java index e15e85e399a..647f73ba14b 100644 --- a/server/src/main/java/io/druid/client/BatchServerInventoryView.java +++ b/server/src/main/java/io/druid/client/BatchServerInventoryView.java @@ -21,7 +21,7 @@ package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; diff --git a/server/src/main/java/io/druid/client/SingleServerInventoryView.java b/server/src/main/java/io/druid/client/SingleServerInventoryView.java index 4f1d91bbf61..b4d71e55ed9 100644 --- a/server/src/main/java/io/druid/client/SingleServerInventoryView.java +++ b/server/src/main/java/io/druid/client/SingleServerInventoryView.java @@ -21,7 +21,7 @@ package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.MapMaker; diff --git a/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategy.java b/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategy.java index 36c3a730447..5eb14fdf6d6 100644 --- a/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategy.java +++ b/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategy.java @@ -2,7 +2,7 @@ package io.druid.client.selector; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; -import com.google.api.client.util.Maps; +import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import java.util.Comparator; diff --git a/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategyConfig.java b/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategyConfig.java index d5bc0201c70..d5e9ee3fb4b 100644 --- a/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategyConfig.java +++ b/server/src/main/java/io/druid/client/selector/CustomTierSelectorStrategyConfig.java @@ -20,7 +20,7 @@ package io.druid.client.selector; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.util.Lists; +import com.google.common.collect.Lists; import java.util.List; diff --git a/server/src/main/java/io/druid/guice/http/JettyHttpClientModule.java b/server/src/main/java/io/druid/guice/http/JettyHttpClientModule.java index 4e77a198873..add7927bf42 100644 --- a/server/src/main/java/io/druid/guice/http/JettyHttpClientModule.java +++ b/server/src/main/java/io/druid/guice/http/JettyHttpClientModule.java @@ -19,7 +19,7 @@ package io.druid.guice.http; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import com.google.inject.Binder; import com.google.inject.Module; import com.metamx.common.lifecycle.Lifecycle; diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java index 605ef526476..2d1fc6b8a7c 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java @@ -21,13 +21,12 @@ package io.druid.metadata; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.api.client.util.Maps; import com.google.common.base.Charsets; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.metamx.common.Pair; import com.metamx.common.RetryUtils; import com.metamx.emitter.EmittingLogger; diff --git a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java index 36a30eea391..fd9ef5cfabd 100644 --- a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java +++ b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java @@ -21,9 +21,9 @@ package io.druid.segment.indexing.granularity; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.util.Lists; import com.google.common.base.Optional; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import com.google.common.collect.PeekingIterator; import com.google.common.collect.Sets; import com.metamx.common.Granularity; diff --git a/server/src/main/java/io/druid/segment/realtime/FireDepartment.java b/server/src/main/java/io/druid/segment/realtime/FireDepartment.java index 446e518aa9a..af6d04a40de 100644 --- a/server/src/main/java/io/druid/segment/realtime/FireDepartment.java +++ b/server/src/main/java/io/druid/segment/realtime/FireDepartment.java @@ -21,19 +21,13 @@ package io.druid.segment.realtime; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.util.Lists; import com.google.common.base.Preconditions; import io.druid.data.input.Firehose; -import io.druid.data.input.FirehoseFactory; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.IngestionSpec; import io.druid.segment.indexing.RealtimeIOConfig; import io.druid.segment.indexing.RealtimeTuningConfig; -import io.druid.segment.indexing.granularity.UniformGranularitySpec; import io.druid.segment.realtime.plumber.Plumber; -import io.druid.segment.realtime.plumber.PlumberSchool; -import io.druid.segment.realtime.plumber.RealtimePlumberSchool; -import org.joda.time.Interval; import java.io.IOException; diff --git a/server/src/main/java/io/druid/segment/realtime/FireHydrant.java b/server/src/main/java/io/druid/segment/realtime/FireHydrant.java index 282ef63b624..2e68451d9a3 100644 --- a/server/src/main/java/io/druid/segment/realtime/FireHydrant.java +++ b/server/src/main/java/io/druid/segment/realtime/FireHydrant.java @@ -19,7 +19,7 @@ package io.druid.segment.realtime; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import io.druid.segment.IncrementalIndexSegment; import io.druid.segment.ReferenceCountingSegment; import io.druid.segment.Segment; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java index faea652273d..62332d2c25c 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java @@ -21,8 +21,8 @@ package io.druid.segment.realtime.firehose; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import com.metamx.emitter.EmittingLogger; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java index d2dcf606ee1..8bffa62956f 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java @@ -21,7 +21,7 @@ package io.druid.segment.realtime.firehose; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.metamx.common.ISE; import com.metamx.emitter.EmittingLogger; diff --git a/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java b/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java index b4853de6d59..12961b4810c 100644 --- a/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java @@ -19,7 +19,7 @@ package io.druid.server.coordinator; -import com.google.api.client.util.Lists; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; diff --git a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java index ea917a1b438..fe42b95b129 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java @@ -19,8 +19,8 @@ package io.druid.server.coordinator.rules; -import com.google.api.client.util.Maps; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.emitter.EmittingLogger; import io.druid.server.coordinator.BalancerStrategy; diff --git a/server/src/main/java/io/druid/server/http/TiersResource.java b/server/src/main/java/io/druid/server/http/TiersResource.java index 541ca5674a1..97810a7cd70 100644 --- a/server/src/main/java/io/druid/server/http/TiersResource.java +++ b/server/src/main/java/io/druid/server/http/TiersResource.java @@ -19,11 +19,11 @@ package io.druid.server.http; -import com.google.api.client.util.Lists; -import com.google.api.client.util.Maps; import com.google.common.base.Function; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.collect.Table; import com.google.inject.Inject; diff --git a/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java index bcd5ba94176..7cceabe8651 100644 --- a/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java +++ b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java @@ -21,7 +21,7 @@ package io.druid.server.router; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Charsets; +import com.google.common.base.Charsets; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.inject.Inject; diff --git a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java index 3d6fdaea13a..2c5aa0ae056 100644 --- a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import io.druid.data.input.InputRow; diff --git a/server/src/test/java/io/druid/server/coordination/CostBalancerStrategyTest.java b/server/src/test/java/io/druid/server/coordination/CostBalancerStrategyTest.java index cd12eaf7317..7f917d8b4c8 100644 --- a/server/src/test/java/io/druid/server/coordination/CostBalancerStrategyTest.java +++ b/server/src/test/java/io/druid/server/coordination/CostBalancerStrategyTest.java @@ -19,7 +19,7 @@ package io.druid.server.coordination; -import com.google.api.client.util.Lists; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.druid.client.ImmutableDruidServer; import io.druid.server.coordinator.BalancerStrategy; @@ -27,11 +27,11 @@ import io.druid.server.coordinator.CostBalancerStrategy; import io.druid.server.coordinator.LoadQueuePeonTester; import io.druid.server.coordinator.ServerHolder; import io.druid.timeline.DataSegment; -import org.junit.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; +import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index 8192cdd0eee..7ae926dee92 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -19,9 +19,9 @@ package io.druid.server.coordinator.rules; -import com.google.api.client.util.Lists; -import com.google.api.client.util.Maps; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; diff --git a/server/src/test/java/io/druid/server/initialization/JettyTest.java b/server/src/test/java/io/druid/server/initialization/JettyTest.java index a95e883b65d..710844d4d5d 100644 --- a/server/src/test/java/io/druid/server/initialization/JettyTest.java +++ b/server/src/test/java/io/druid/server/initialization/JettyTest.java @@ -19,7 +19,7 @@ package io.druid.server.initialization; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Binder; @@ -33,12 +33,12 @@ import com.metamx.http.client.HttpClient; import com.metamx.http.client.response.InputStreamResponseHandler; import com.metamx.http.client.response.StatusResponseHandler; import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.guice.GuiceInjectors; import io.druid.guice.Jerseys; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; import io.druid.guice.annotations.Global; -import io.druid.guice.GuiceInjectors; import io.druid.guice.annotations.Self; import io.druid.initialization.Initialization; import io.druid.server.DruidNode; diff --git a/services/src/main/java/io/druid/cli/CliInternalHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliInternalHadoopIndexer.java index 18a35da9743..9b310b35b08 100644 --- a/services/src/main/java/io/druid/cli/CliInternalHadoopIndexer.java +++ b/services/src/main/java/io/druid/cli/CliInternalHadoopIndexer.java @@ -19,11 +19,11 @@ package io.druid.cli; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; -import com.google.api.client.repackaged.com.google.common.base.Throwables; -import com.google.api.client.util.Lists; +import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; diff --git a/services/src/main/java/io/druid/cli/PullDependencies.java b/services/src/main/java/io/druid/cli/PullDependencies.java index e48dd7658bc..c2efe844d5f 100644 --- a/services/src/main/java/io/druid/cli/PullDependencies.java +++ b/services/src/main/java/io/druid/cli/PullDependencies.java @@ -19,8 +19,8 @@ package io.druid.cli; -import com.google.api.client.util.Lists; import com.google.common.base.Throwables; +import com.google.common.collect.Lists; import com.google.inject.Inject; import io.airlift.command.Command; import io.airlift.command.Option; diff --git a/services/src/main/java/io/druid/cli/convert/DataSegmentPusherDefaultConverter.java b/services/src/main/java/io/druid/cli/convert/DataSegmentPusherDefaultConverter.java index 5484320b53d..4059e10b612 100644 --- a/services/src/main/java/io/druid/cli/convert/DataSegmentPusherDefaultConverter.java +++ b/services/src/main/java/io/druid/cli/convert/DataSegmentPusherDefaultConverter.java @@ -19,8 +19,8 @@ package io.druid.cli.convert; -import com.google.api.client.util.Maps; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.Map; diff --git a/services/src/main/java/io/druid/cli/convert/DatabasePropertiesConverter.java b/services/src/main/java/io/druid/cli/convert/DatabasePropertiesConverter.java index a46d5cd6e88..cba415cc891 100644 --- a/services/src/main/java/io/druid/cli/convert/DatabasePropertiesConverter.java +++ b/services/src/main/java/io/druid/cli/convert/DatabasePropertiesConverter.java @@ -19,9 +19,9 @@ package io.druid.cli.convert; -import com.google.api.client.util.Maps; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.util.List; import java.util.Map; diff --git a/services/src/main/java/io/druid/cli/convert/PrefixRename.java b/services/src/main/java/io/druid/cli/convert/PrefixRename.java index 3e8cd67c217..1d3b4f73818 100644 --- a/services/src/main/java/io/druid/cli/convert/PrefixRename.java +++ b/services/src/main/java/io/druid/cli/convert/PrefixRename.java @@ -19,8 +19,8 @@ package io.druid.cli.convert; -import com.google.api.client.util.Maps; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.Map; import java.util.Properties; diff --git a/services/src/main/java/io/druid/cli/validate/DruidJsonValidator.java b/services/src/main/java/io/druid/cli/validate/DruidJsonValidator.java index 2784fe0c7a5..92c69a2efc2 100644 --- a/services/src/main/java/io/druid/cli/validate/DruidJsonValidator.java +++ b/services/src/main/java/io/druid/cli/validate/DruidJsonValidator.java @@ -20,7 +20,7 @@ package io.druid.cli.validate; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; import com.metamx.common.UOE; import io.airlift.command.Command; import io.airlift.command.Option; From b6f71d3fd6c4f4b0939811a88e563b31af831118 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 3 Dec 2014 11:11:53 -0800 Subject: [PATCH 10/14] Fix confusing error message in HadoopIndexTask --- .../java/io/druid/indexing/common/task/HadoopIndexTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index 7087d0dbfcb..24b6d39e8dd 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -117,7 +117,7 @@ public class HadoopIndexTask extends AbstractTask Preconditions.checkArgument(this.spec.getTuningConfig().getWorkingPath() == null, "workingPath must be absent"); Preconditions.checkArgument( this.spec.getIOConfig().getMetadataUpdateSpec() == null, - "updaterJobSpec must be absent" + "metadataUpdateSpec must be absent" ); if (hadoopDependencyCoordinates != null) { From 325a5c4abc3248395dd22cb32f86e9dc84f411a2 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 3 Dec 2014 13:18:33 -0800 Subject: [PATCH 11/14] Update ForkingTaskRunner to remove @Deprecated Files method usage --- .../main/java/io/druid/indexing/overlord/ForkingTaskRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java index 01696f86bef..5443a9b39a7 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java @@ -234,7 +234,7 @@ public class ForkingTaskRunner implements TaskRunner, TaskLogStreamer log.info("Logging task %s output to: %s", task.getId(), logFile); boolean runFailed = true; - try (final OutputStream toLogfile = Files.newOutputStreamSupplier(logFile).getOutput()) { + try (final OutputStream toLogfile = Files.asByteSink(logFile).openBufferedStream()) { ByteStreams.copy(processHolder.process.getInputStream(), toLogfile); final int statusCode = processHolder.process.waitFor(); log.info("Process exited with status[%d] for task: %s", statusCode, task.getId()); From 2e6c25493738e45daf4b71fc852a6c3d62cd73f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 3 Dec 2014 15:09:31 -0800 Subject: [PATCH 12/14] metadata injection not needed for indexing service --- .../io/druid/indexer/HadoopDruidIndexerJob.java | 6 ++++++ .../indexing/common/task/HadoopIndexTask.java | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java index 32cd0f4d3f1..decf9d943bf 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java @@ -19,6 +19,7 @@ package io.druid.indexer; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.metamx.common.logger.Logger; @@ -45,6 +46,11 @@ public class HadoopDruidIndexerJob implements Jobby config.verify(); this.config = config; + Preconditions.checkArgument( + !config.isUpdaterJobSpecSet() || handler != null, + "MetadataStorageUpdaterJobHandler must not be null if ioConfig.metadataUpdateSpec is specified in " + ); + if (config.isUpdaterJobSpecSet()) { metadataStorageUpdaterJob = new MetadataStorageUpdaterJob( config, diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index 2b032f47730..f195872dc55 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -290,10 +290,16 @@ public class HadoopIndexTask extends AbstractTask .withTuningConfig(theSchema.getTuningConfig().withVersion(version)) ); - HadoopDruidIndexerJob job = new HadoopDruidIndexerJob( - config, - injector.getInstance(MetadataStorageUpdaterJobHandler.class) - ); + // MetadataStorageUpdaterJobHandler is only needed when running standalone without indexing service + // In that case the whatever runs the Hadoop Index Task must ensure MetadataStorageUpdaterJobHandler + // can be injected based on the configuration given in config.getSchema().getIOConfig().getMetadataUpdateSpec() + final MetadataStorageUpdaterJobHandler maybeHandler; + if (config.isUpdaterJobSpecSet()) { + maybeHandler = injector.getInstance(MetadataStorageUpdaterJobHandler.class); + } else { + maybeHandler = null; + } + HadoopDruidIndexerJob job = new HadoopDruidIndexerJob(config, maybeHandler); log.info("Starting a hadoop index generator job..."); if (job.run()) { From c867d59ee047da7ea4308be01ef4b0789f3e7bc3 Mon Sep 17 00:00:00 2001 From: xvrl Date: Wed, 3 Dec 2014 15:30:32 -0800 Subject: [PATCH 13/14] fix error message --- .../src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java index decf9d943bf..49ea3654738 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerJob.java @@ -48,7 +48,7 @@ public class HadoopDruidIndexerJob implements Jobby Preconditions.checkArgument( !config.isUpdaterJobSpecSet() || handler != null, - "MetadataStorageUpdaterJobHandler must not be null if ioConfig.metadataUpdateSpec is specified in " + "MetadataStorageUpdaterJobHandler must not be null if ioConfig.metadataUpdateSpec is specified." ); if (config.isUpdaterJobSpecSet()) { From 187126dd4be89f8657d38b140652e829d0e0d8a8 Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 4 Dec 2014 10:49:36 -0800 Subject: [PATCH 14/14] fix broken unit tests --- .../client/CachingClusteredClientTest.java | 12 +++++++ .../druid/client/CachingQueryRunnerTest.java | 33 ++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 7e17d4ac65e..4220de989d3 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -1902,6 +1902,18 @@ public class CachingClusteredClientTest jsonMapper, new CacheConfig() { + @Override + public boolean isPopulateCache() + { + return true; + } + + @Override + public boolean isUseCache() + { + return true; + } + @Override public boolean isQueryCacheable(Query query) { diff --git a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java index 4975e0b5f01..6252e9b7bcb 100644 --- a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java +++ b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java @@ -82,7 +82,7 @@ public class CachingQueryRunnerTest @Test public void testCloseAndPopulate() throws Exception { - Iterable> expectedRes = makeTopNResults(false,objects); + Iterable> expectedRes = makeTopNResults(false, objects); final TopNQueryBuilder builder = new TopNQueryBuilder() .dataSource("ds") .dimension("top_dim") @@ -130,7 +130,19 @@ public class CachingQueryRunnerTest } }, new CacheConfig() + { + @Override + public boolean isPopulateCache() + { + return true; + } + @Override + public boolean isUseCache() + { + return true; + } + } ); TopNQuery query = builder.build(); @@ -141,7 +153,7 @@ public class CachingQueryRunnerTest cacheStrategy.computeCacheKey(query) ); - HashMap context = new HashMap(); + HashMap context = new HashMap(); Sequence res = runner.run(query, context); // base sequence is not closed yet Assert.assertFalse("sequence must not be closed", closable.isClosed()); @@ -220,9 +232,22 @@ public class CachingQueryRunnerTest } }, new CacheConfig() + { + @Override + public boolean isPopulateCache() + { + return true; + } + + @Override + public boolean isUseCache() + { + return true; + } + } ); - HashMap context = new HashMap(); + HashMap context = new HashMap(); List results = Sequences.toList(runner.run(query, context), new ArrayList()); Assert.assertEquals(expectedResults, results); } @@ -252,7 +277,7 @@ public class CachingQueryRunnerTest "rows", rows, "imps", imps, "impers", imps - ) + ) ); } else { values.add(