Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MSQ: Use long instead of double for estimatedRetainedBytes. (#13272) 

Fixes a problem where, due to the inexactness of floating-point math, we
would potentially drift while tracking retained byte counts and run into
assertion failures in assertRetainedByteCountsAreTrackedCorrectly.
This commit is contained in:
Gian Merlino 2022-10-28 08:31:52 -07:00 committed by GitHub
parent 5429b9d764
commit 4f0145fb85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/ClusterByStatisticsCollectorImpl.java
View File

@ -58,7 +58,7 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
private final int maxRetainedBytes;
private final int maxBuckets;
private double totalRetainedBytes;
private long totalRetainedBytes;
private ClusterByStatisticsCollectorImpl(
final ClusterBy clusterBy,
@ -369,8 +369,8 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
*/
private void downSample()
{
double newTotalRetainedBytes = totalRetainedBytes;
final double targetTotalRetainedBytes = totalRetainedBytes / 2;
long newTotalRetainedBytes = totalRetainedBytes;
final long targetTotalRetainedBytes = totalRetainedBytes / 2;
final List<BucketHolder> sortedHolders = new ArrayList<>(buckets.size());
@ -384,7 +384,8 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
// Downsample least-dense buckets first. (They're less likely to need high resolution.)
sortedHolders.sort(
Comparator.comparing((BucketHolder holder) ->
(double) holder.keyCollector.estimatedTotalWeight() / holder.keyCollector.estimatedRetainedKeys())
(double) holder.keyCollector.estimatedTotalWeight()
/ holder.keyCollector.estimatedRetainedKeys())
);
int i = 0;

2
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DelegateOrMinKeyCollector.java
View File

@ -128,7 +128,7 @@ public class DelegateOrMinKeyCollector<TDelegate extends KeyCollector<TDelegate>
}
@Override
public double estimatedRetainedBytes()
public long estimatedRetainedBytes()
{
if (delegate != null) {
return delegate.estimatedRetainedBytes();

2
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
View File

@ -172,7 +172,7 @@ public class DistinctKeyCollector implements KeyCollector<DistinctKeyCollector>
}
@Override
public double estimatedRetainedBytes()
public long estimatedRetainedBytes()
{
return retainedBytes;
}

2
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/KeyCollector.java
View File

@ -57,7 +57,7 @@ public interface KeyCollector<CollectorType extends KeyCollector<CollectorType>>
* Returns an estimate of the number of bytes currently retained by this collector. This may change over time as
* more keys are added.
*/
double estimatedRetainedBytes();
long estimatedRetainedBytes();
/**
* Downsample this collector, dropping about half of the keys that are currently retained. Returns true if

4
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/QuantilesSketchKeyCollector.java
View File

@ -102,9 +102,9 @@ public class QuantilesSketchKeyCollector implements KeyCollector<QuantilesSketch
}
@Override
public double estimatedRetainedBytes()
public long estimatedRetainedBytes()
{
return averageKeyLength * estimatedRetainedKeys();
return Math.round(averageKeyLength * estimatedRetainedKeys());
}
@Override