Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Log merge and push timings for PartialGenericSegmentMergeTask (#14089)

This commit is contained in:
Rohan Garg 2023-04-18 11:51:26 +05:30 committed by GitHub
parent f6a0888bc0
commit 086b2b8efe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentMergeTask.java
View File

@ -254,6 +254,7 @@ abstract class PartialSegmentMergeTask<S extends ShardSpec> extends PerfectRollu
for (Entry<Interval, Int2ObjectMap<List<File>>> entryPerInterval : intervalToUnzippedFiles.entrySet()) { for (Entry<Interval, Int2ObjectMap<List<File>>> entryPerInterval : intervalToUnzippedFiles.entrySet()) {
final Interval interval = entryPerInterval.getKey(); final Interval interval = entryPerInterval.getKey();
for (Int2ObjectMap.Entry<List<File>> entryPerBucketId : entryPerInterval.getValue().int2ObjectEntrySet()) { for (Int2ObjectMap.Entry<List<File>> entryPerBucketId : entryPerInterval.getValue().int2ObjectEntrySet()) {
long startTime = System.nanoTime();
final int bucketId = entryPerBucketId.getIntKey(); final int bucketId = entryPerBucketId.getIntKey();
final List<File> segmentFilesToMerge = entryPerBucketId.getValue(); final List<File> segmentFilesToMerge = entryPerBucketId.getValue();
final Pair<File, List<String>> mergedFileAndDimensionNames = mergeSegmentsInSamePartition( final Pair<File, List<String>> mergedFileAndDimensionNames = mergeSegmentsInSamePartition(
@ -266,6 +267,12 @@ abstract class PartialSegmentMergeTask<S extends ShardSpec> extends PerfectRollu
persistDir, persistDir,
0 0
); );
long mergeFinishTime = System.nanoTime();
LOG.info("Merged [%d] input segment(s) for interval [%s] in [%,d]ms.",
segmentFilesToMerge.size(),
interval,
(mergeFinishTime - startTime) / 1000000
);
final List<String> metricNames = Arrays.stream(dataSchema.getAggregators()) final List<String> metricNames = Arrays.stream(dataSchema.getAggregators())
.map(AggregatorFactory::getName) .map(AggregatorFactory::getName)
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -294,7 +301,18 @@ abstract class PartialSegmentMergeTask<S extends ShardSpec> extends PerfectRollu
exception -> !(exception instanceof NullPointerException) && exception instanceof Exception, exception -> !(exception instanceof NullPointerException) && exception instanceof Exception,
5 5
); );
long pushFinishTime = System.nanoTime();
pushedSegments.add(segment); pushedSegments.add(segment);
LOG.info("Built segment [%s] for interval [%s] (from [%d] input segment(s) in [%,d]ms) of "
+ "size [%d] bytes and pushed ([%,d]ms) to deep storage [%s].",
segment.getId(),
interval,
segmentFilesToMerge.size(),
(mergeFinishTime - startTime) / 1000000,
segment.getSize(),
(pushFinishTime - mergeFinishTime) / 1000000,
segment.getLoadSpec()
);
} }
} }
return pushedSegments; return pushedSegments;