From 929b981710c6845685701c7df6f1a1a9cf92c429 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Thu, 5 Nov 2015 18:09:47 -0800 Subject: [PATCH 1/2] Change DefaultObjectMapper to NOT overwrite final fields unless explicitly asked to --- .../io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java | 2 +- .../src/main/java/io/druid/indexer/HadoopTuningConfig.java | 2 +- .../src/main/java/io/druid/indexing/common/task/MoveTask.java | 2 +- .../src/main/java/io/druid/jackson/DefaultObjectMapper.java | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java index 2c715700003..2cefd8bd3ca 100644 --- a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java +++ b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java @@ -123,7 +123,7 @@ public class RabbitMQFirehoseFactory implements FirehoseFactory getTargetLoadSpec() { return targetLoadSpec; diff --git a/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java index b1d298d5cb0..d1fc68ae30c 100644 --- a/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java +++ b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java @@ -53,6 +53,7 @@ public class DefaultObjectMapper extends ObjectMapper configure(MapperFeature.AUTO_DETECT_FIELDS, false); configure(MapperFeature.AUTO_DETECT_IS_GETTERS, false); configure(MapperFeature.AUTO_DETECT_SETTERS, false); + configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false); configure(SerializationFeature.INDENT_OUTPUT, false); } From abae47850a99ca7345bde6a9b053b397524e1f8e Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 11 Nov 2015 10:27:00 -0800 Subject: [PATCH 2/2] Add backwards compatability for PR #1922 --- .../rabbitmq/RabbitMQFirehoseFactory.java | 6 ++++-- .../rabbitmq/RabbitMQFirehoseFactoryTest.java | 6 ++++-- .../io/druid/indexer/HadoopTuningConfig.java | 18 ++++++++++++------ .../druid/indexer/BatchDeltaIngestionTest.java | 3 ++- .../DetermineHashedPartitionsJobTest.java | 3 ++- .../indexer/DeterminePartitionsJobTest.java | 3 ++- .../indexer/HadoopDruidIndexerConfigTest.java | 3 ++- .../druid/indexer/HadoopTuningConfigTest.java | 3 ++- .../druid/indexer/IndexGeneratorJobTest.java | 3 ++- .../java/io/druid/indexer/JobHelperTest.java | 3 ++- .../updater/HadoopConverterJobTest.java | 3 ++- .../druid/indexing/common/task/MoveTask.java | 6 ++++-- .../indexing/common/task/TaskSerdeTest.java | 1 + 13 files changed, 41 insertions(+), 20 deletions(-) diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java index 2cefd8bd3ca..c4b80015ddb 100644 --- a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java +++ b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java @@ -107,11 +107,13 @@ public class RabbitMQFirehoseFactory implements FirehoseFactory jobProperties, final @JsonProperty("combineText") boolean combineText, - final @JsonProperty("useCombiner") Boolean useCombiner + final @JsonProperty("useCombiner") Boolean useCombiner, + // See https://github.com/druid-io/druid/pull/1922 + final @JsonProperty("rowFlushBoundary") Integer maxRowsInMemoryCOMPAT ) { this.workingPath = workingPath; @@ -96,7 +99,7 @@ public class HadoopTuningConfig implements TuningConfig this.partitionsSpec = partitionsSpec == null ? DEFAULT_PARTITIONS_SPEC : partitionsSpec; this.shardSpecs = shardSpecs == null ? DEFAULT_SHARD_SPECS : shardSpecs; this.indexSpec = indexSpec == null ? DEFAULT_INDEX_SPEC : indexSpec; - this.rowFlushBoundary = maxRowsInMemory == null ? DEFAULT_ROW_FLUSH_BOUNDARY : maxRowsInMemory; + this.rowFlushBoundary = maxRowsInMemory == null ? maxRowsInMemoryCOMPAT == null ? DEFAULT_ROW_FLUSH_BOUNDARY : maxRowsInMemoryCOMPAT : maxRowsInMemory; this.leaveIntermediate = leaveIntermediate; this.cleanupOnFailure = cleanupOnFailure == null ? true : cleanupOnFailure; this.overwriteFiles = overwriteFiles; @@ -201,7 +204,8 @@ public class HadoopTuningConfig implements TuningConfig ignoreInvalidRows, jobProperties, combineText, - useCombiner + useCombiner, + null ); } @@ -220,7 +224,8 @@ public class HadoopTuningConfig implements TuningConfig ignoreInvalidRows, jobProperties, combineText, - useCombiner + useCombiner, + null ); } @@ -239,7 +244,8 @@ public class HadoopTuningConfig implements TuningConfig ignoreInvalidRows, jobProperties, combineText, - useCombiner + useCombiner, + null ); } } diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java index 3350041abe5..2f5183a2571 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/BatchDeltaIngestionTest.java @@ -380,7 +380,8 @@ public class BatchDeltaIngestionTest false, null, false, - false + false, + null ) ) ); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/DetermineHashedPartitionsJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/DetermineHashedPartitionsJobTest.java index 2219bb837a4..de1b6d6eddd 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/DetermineHashedPartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/DetermineHashedPartitionsJobTest.java @@ -160,7 +160,8 @@ public class DetermineHashedPartitionsJobTest false, null, false, - false + false, + null ) ); this.indexerConfig = new HadoopDruidIndexerConfig(ingestionSpec); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/DeterminePartitionsJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/DeterminePartitionsJobTest.java index dc1e4722be1..9632dd1d5f4 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/DeterminePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/DeterminePartitionsJobTest.java @@ -263,7 +263,8 @@ public class DeterminePartitionsJobTest false, null, false, - false + false, + null ) ) ); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java index 7bb88bc4e36..e85acf3437b 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -204,7 +204,8 @@ public class HadoopDruidIndexerConfigTest false, null, false, - false + false, + null ) ); HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec(spec); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopTuningConfigTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopTuningConfigTest.java index 78d4e44e165..cd69fb7bb17 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopTuningConfigTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopTuningConfigTest.java @@ -52,7 +52,8 @@ public class HadoopTuningConfigTest true, null, true, - true + true, + null ); HadoopTuningConfig actual = jsonReadWriteRead(jsonMapper.writeValueAsString(expected), HadoopTuningConfig.class); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java index 361ac646d75..e72acc86822 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java @@ -394,7 +394,8 @@ public class IndexGeneratorJobTest false, ImmutableMap.of(JobContext.NUM_REDUCES, "0"), //verifies that set num reducers is ignored false, - useCombiner + useCombiner, + null ) ) ); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/JobHelperTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/JobHelperTest.java index 1c4294c2ca0..367a05a7ea5 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/JobHelperTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/JobHelperTest.java @@ -112,7 +112,8 @@ public class JobHelperTest "THISISMYACCESSKEY" ), false, - false + false, + null ) ) ); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java index b7705c89d15..4040537de7e 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java @@ -200,7 +200,8 @@ public class HadoopConverterJobTest false, null, false, - false + false, + null ) ) ); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java index 7b43f574c8d..9bf5fa0a187 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MoveTask.java @@ -46,7 +46,9 @@ public class MoveTask extends AbstractFixedIntervalTask @JsonProperty("dataSource") String dataSource, @JsonProperty("interval") Interval interval, @JsonProperty("target") Map targetLoadSpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context, + // See https://github.com/druid-io/druid/pull/1922 + @JsonProperty("targetLoadSpec") Map targetLoadSpecCOMPAT ) { super( @@ -55,7 +57,7 @@ public class MoveTask extends AbstractFixedIntervalTask interval, context ); - this.targetLoadSpec = targetLoadSpec; + this.targetLoadSpec = targetLoadSpec == null ? targetLoadSpecCOMPAT : targetLoadSpec; } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java index ca4e267e208..56456cd84e0 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java @@ -501,6 +501,7 @@ public class TaskSerdeTest "foo", new Interval("2010-01-01/P1D"), ImmutableMap.of("bucket", "hey", "baseKey", "what"), + null, null );