Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ensure ReferenceCountingSegment.decrement() is invoked correctly (#8323) 

* fix issue8291. Make sure ReferenceCountingSegment.decrement() is invoked correctly

* add some comments in SinkQuerySegmentWalker.java

* extracting perSegmentRunners and perHydrantRunners to reduce the level of nesting
This commit is contained in:
pphust 2019-08-21 03:01:16 +08:00 committed by Roman Leventov
parent 818bf4990c
commit ffcbd1ecb8
1 changed files with 88 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
server/src/main/java/org/apache/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java
View File

@ -177,13 +177,8 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
final boolean skipIncrementalSegment = query.getContextValue(CONTEXT_SKIP_INCREMENTAL_SEGMENT, false); final boolean skipIncrementalSegment = query.getContextValue(CONTEXT_SKIP_INCREMENTAL_SEGMENT, false);
final AtomicLong cpuTimeAccumulator = new AtomicLong(0L); final AtomicLong cpuTimeAccumulator = new AtomicLong(0L);
final QueryRunner<T> mergedRunner = Iterable<QueryRunner<T>> perSegmentRunners = Iterables.transform(
toolChest.mergeResults( specs,
factory.mergeRunners(
queryExecutorService,
FunctionalIterable
.create(specs)
.transform(
descriptor -> { descriptor -> {
final PartitionHolder<Sink> holder = sinkTimeline.findEntry( final PartitionHolder<Sink> holder = sinkTimeline.findEntry(
descriptor.getInterval(), descriptor.getInterval(),
@ -201,14 +196,7 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
final Sink theSink = chunk.getObject(); final Sink theSink = chunk.getObject();
final SegmentId sinkSegmentId = theSink.getSegment().getId(); final SegmentId sinkSegmentId = theSink.getSegment().getId();
return new SpecificSegmentQueryRunner<>( Iterable<QueryRunner<T>> perHydrantRunners = new SinkQueryRunners<>(
withPerSinkMetrics(
new BySegmentQueryRunner<>(
sinkSegmentId,
descriptor.getInterval().getStart(),
factory.mergeRunners(
Execs.directExecutor(),
new SinkQueryRunners<>(
Iterables.transform( Iterables.transform(
theSink, theSink,
hydrant -> { hydrant -> {
@ -221,23 +209,20 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
} }
// Prevent the underlying segment from swapping when its being iterated // Prevent the underlying segment from swapping when its being iterated
final Pair<Segment, Closeable> segment = hydrant.getAndIncrementSegment(); final Pair<Segment, Closeable> segmentAndCloseable = hydrant.getAndIncrementSegment();
try { try {
QueryRunner<T> baseRunner = QueryRunnerHelper.makeClosingQueryRunner( QueryRunner<T> runner = factory.createRunner(segmentAndCloseable.lhs);
factory.createRunner(segment.lhs),
segment.rhs
);
// 1) Only use caching if data is immutable // 1) Only use caching if data is immutable
// 2) Hydrants are not the same between replicas, make sure cache is local // 2) Hydrants are not the same between replicas, make sure cache is local
if (hydrantDefinitelySwapped && cache.isLocal()) { if (hydrantDefinitelySwapped && cache.isLocal()) {
QueryRunner<T> cachingRunner = new CachingQueryRunner<>( runner = new CachingQueryRunner<>(
makeHydrantCacheIdentifier(hydrant), makeHydrantCacheIdentifier(hydrant),
descriptor, descriptor,
objectMapper, objectMapper,
cache, cache,
toolChest, toolChest,
baseRunner, runner,
// Always populate in foreground regardless of config // Always populate in foreground regardless of config
new ForegroundCachePopulator( new ForegroundCachePopulator(
objectMapper, objectMapper,
@ -246,18 +231,29 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
), ),
cacheConfig cacheConfig
); );
return new Pair<>(segment.lhs.getDataInterval(), cachingRunner);
} else {
return new Pair<>(segment.lhs.getDataInterval(), baseRunner);
} }
// Make it always use Closeable to decrement()
runner = QueryRunnerHelper.makeClosingQueryRunner(
runner,
segmentAndCloseable.rhs
);
return new Pair<>(segmentAndCloseable.lhs.getDataInterval(), runner);
} }
catch (RuntimeException e) { catch (RuntimeException e) {
CloseQuietly.close(segment.rhs); CloseQuietly.close(segmentAndCloseable.rhs);
throw e; throw e;
} }
} }
) )
) );
return new SpecificSegmentQueryRunner<>(
withPerSinkMetrics(
new BySegmentQueryRunner<>(
sinkSegmentId,
descriptor.getInterval().getStart(),
factory.mergeRunners(
Execs.directExecutor(),
perHydrantRunners
) )
), ),
toolChest, toolChest,
@ -267,7 +263,12 @@ public class SinkQuerySegmentWalker implements QuerySegmentWalker
new SpecificSegmentSpec(descriptor) new SpecificSegmentSpec(descriptor)
); );
} }
) );
final QueryRunner<T> mergedRunner =
toolChest.mergeResults(
factory.mergeRunners(
queryExecutorService,
perSegmentRunners
) )
); );