diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java index 74e9b6295be..829a03a6844 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java @@ -88,6 +88,7 @@ public class DetermineHashedPartitionsJob implements Jobby ); JobHelper.injectSystemProperties(groupByJob); + config.addJobProperties(groupByJob); groupByJob.setMapperClass(DetermineCardinalityMapper.class); groupByJob.setMapOutputKeyClass(LongWritable.class); groupByJob.setMapOutputValueClass(BytesWritable.class); @@ -104,7 +105,6 @@ public class DetermineHashedPartitionsJob implements Jobby JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()), groupByJob); config.addInputPaths(groupByJob); - config.addJobProperties(groupByJob); config.intoConfiguration(groupByJob); FileOutputFormat.setOutputPath(groupByJob, config.makeGroupedDataDir()); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java index 0060303888f..a502c340c1b 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java @@ -126,6 +126,8 @@ public class DeterminePartitionsJob implements Jobby ); JobHelper.injectSystemProperties(groupByJob); + config.addJobProperties(groupByJob); + groupByJob.setMapperClass(DeterminePartitionsGroupByMapper.class); groupByJob.setMapOutputKeyClass(BytesWritable.class); groupByJob.setMapOutputValueClass(NullWritable.class); @@ -137,7 +139,6 @@ public class DeterminePartitionsJob implements Jobby JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()), groupByJob); config.addInputPaths(groupByJob); - config.addJobProperties(groupByJob); config.intoConfiguration(groupByJob); FileOutputFormat.setOutputPath(groupByJob, config.makeGroupedDataDir()); @@ -163,6 +164,7 @@ public class DeterminePartitionsJob implements Jobby dimSelectionJob.getConfiguration().set("io.sort.record.percent", "0.19"); JobHelper.injectSystemProperties(dimSelectionJob); + config.addJobProperties(dimSelectionJob); if (!config.getPartitionsSpec().isAssumeGrouped()) { // Read grouped data from the groupByJob. @@ -186,7 +188,6 @@ public class DeterminePartitionsJob implements Jobby dimSelectionJob.setNumReduceTasks(config.getGranularitySpec().bucketIntervals().get().size()); JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()), dimSelectionJob); - config.addJobProperties(dimSelectionJob); config.intoConfiguration(dimSelectionJob); FileOutputFormat.setOutputPath(dimSelectionJob, config.makeIntermediatePath()); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index b54eec9d185..16de192049f 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -148,6 +148,7 @@ public class IndexGeneratorJob implements Jobby job.getConfiguration().set("io.sort.record.percent", "0.23"); JobHelper.injectSystemProperties(job); + config.addJobProperties(job); job.setMapperClass(IndexGeneratorMapper.class); job.setMapOutputValueClass(BytesWritable.class); @@ -173,7 +174,6 @@ public class IndexGeneratorJob implements Jobby job.setOutputFormatClass(IndexGeneratorOutputFormat.class); FileOutputFormat.setOutputPath(job, config.makeIntermediatePath()); - config.addJobProperties(job); config.addInputPaths(job); // hack to get druid.processing.bitmap property passed down to hadoop job. diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java index e64f65adedf..b8ebbb05070 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorJobTest.java @@ -52,6 +52,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.joda.time.DateTime; import org.joda.time.DateTimeComparator; @@ -362,7 +363,7 @@ public class IndexGeneratorJobTest false, false, false, - null, + ImmutableMap.of(JobContext.NUM_REDUCES, "0"), //verifies that set num reducers is ignored false, false, false,