From e8727dc98b4125dcdd35b44cb12a791d404d926a Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 9 Aug 2019 16:03:30 -0500 Subject: [PATCH] Fix DeterminePartitionsJob reducer when total rows < targetPartitionSize * SHARD_COMBINE_THRESHOLD (#8273) * Fix DeterminePartitionsJob reducer when rows < targetPartitionSize * use isEmpty() --- .../druid/indexer/DeterminePartitionsJob.java | 5 ++-- .../indexer/DeterminePartitionsJobTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java index 984d104a80c..f9c4fe8f980 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java @@ -691,8 +691,9 @@ public class DeterminePartitionsJob implements Jobby // One more shard to go final ShardSpec shardSpec; - if (currentDimPartition.rows < config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD) { - // Combine with previous shard + if (currentDimPartition.rows < config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD && + !currentDimPartitions.partitions.isEmpty()) { + // Combine with previous shard if it exists and the current shard is small enough final DimPartition previousDimPartition = currentDimPartitions.partitions.remove( currentDimPartitions.partitions.size() - 1 ); diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java index 2c35b75e251..35174f56dfd 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java @@ -191,6 +191,30 @@ public class DeterminePartitionsJobTest "2014102200,j.example.com,US,333", "2014102200,k.example.com,US,555" ) + }, + { + true, + 1000, + "2014-10-22T00:00:00Z/P1D", + 1, + new int[]{1}, + new String[][][]{ + { + {null, null} + } + }, + ImmutableList.of( + "2014102200,a.example.com,CN,100", + "2014102200,b.exmaple.com,US,50", + "2014102200,c.example.com,US,200", + "2014102200,d.example.com,US,250", + "2014102200,e.example.com,US,123", + "2014102200,f.example.com,US,567", + "2014102200,g.example.com,US,11", + "2014102200,h.example.com,US,251", + "2014102200,i.example.com,US,963", + "2014102200,j.example.com,US,333" + ) } } );