Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix DeterminePartitionsJob reducer when total rows < targetPartitionSize * SHARD_COMBINE_THRESHOLD (#8273) 

* Fix DeterminePartitionsJob reducer when rows < targetPartitionSize

* use isEmpty()
This commit is contained in:
Jonathan Wei 2019-08-09 16:03:30 -05:00 committed by GitHub
parent 170368999d
commit e8727dc98b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java
View File

@ -691,8 +691,9 @@ public class DeterminePartitionsJob implements Jobby
// One more shard to go
final ShardSpec shardSpec;
if (currentDimPartition.rows < config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD) {
// Combine with previous shard
if (currentDimPartition.rows < config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD &&
!currentDimPartitions.partitions.isEmpty()) {
// Combine with previous shard if it exists and the current shard is small enough
final DimPartition previousDimPartition = currentDimPartitions.partitions.remove(
currentDimPartitions.partitions.size() - 1
);

24
indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java
View File

@ -191,6 +191,30 @@ public class DeterminePartitionsJobTest
"2014102200,j.example.com,US,333",
"2014102200,k.example.com,US,555"
)
},
{
true,
1000,
"2014-10-22T00:00:00Z/P1D",
1,
new int[]{1},
new String[][][]{
{
{null, null}
}
},
ImmutableList.of(
"2014102200,a.example.com,CN,100",
"2014102200,b.exmaple.com,US,50",
"2014102200,c.example.com,US,200",
"2014102200,d.example.com,US,250",
"2014102200,e.example.com,US,123",
"2014102200,f.example.com,US,567",
"2014102200,g.example.com,US,11",
"2014102200,h.example.com,US,251",
"2014102200,i.example.com,US,963",
"2014102200,j.example.com,US,333"
)
}
}
);