Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Prevent multiple attempts to publish segments for the same sequence (#14995) 

* Prevent a race that may cause multiple attempts to publish segments for the same sequence
This commit is contained in:
AmatyaAvadhanula 2023-11-16 14:21:26 +05:30 committed by GitHub
parent 857b8de425
commit cdc192d38d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java
View File

@ -213,6 +213,7 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
private final String stream; private final String stream;
private final Set<String> publishingSequences = Sets.newConcurrentHashSet(); private final Set<String> publishingSequences = Sets.newConcurrentHashSet();
private final Set<String> publishedSequences = Sets.newConcurrentHashSet();
private final List<ListenableFuture<SegmentsAndCommitMetadata>> publishWaitList = new ArrayList<>(); private final List<ListenableFuture<SegmentsAndCommitMetadata>> publishWaitList = new ArrayList<>();
private final List<ListenableFuture<SegmentsAndCommitMetadata>> handOffWaitList = new ArrayList<>(); private final List<ListenableFuture<SegmentsAndCommitMetadata>> handOffWaitList = new ArrayList<>();
@ -806,7 +807,8 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
List<SequenceMetadata<PartitionIdType, SequenceOffsetType>> sequencesSnapshot = new ArrayList<>(sequences); List<SequenceMetadata<PartitionIdType, SequenceOffsetType>> sequencesSnapshot = new ArrayList<>(sequences);
for (int i = 0; i < sequencesSnapshot.size(); i++) { for (int i = 0; i < sequencesSnapshot.size(); i++) {
final SequenceMetadata<PartitionIdType, SequenceOffsetType> sequenceMetadata = sequencesSnapshot.get(i); final SequenceMetadata<PartitionIdType, SequenceOffsetType> sequenceMetadata = sequencesSnapshot.get(i);
if (!publishingSequences.contains(sequenceMetadata.getSequenceName())) { if (!publishingSequences.contains(sequenceMetadata.getSequenceName())
&& !publishedSequences.contains(sequenceMetadata.getSequenceName())) {
final boolean isLast = i == (sequencesSnapshot.size() - 1); final boolean isLast = i == (sequencesSnapshot.size() - 1);
if (isLast) { if (isLast) {
// Shorten endOffsets of the last sequence to match currOffsets. // Shorten endOffsets of the last sequence to match currOffsets.
@ -1009,6 +1011,7 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
); );
log.infoSegments(publishedSegmentsAndCommitMetadata.getSegments(), "Published segments"); log.infoSegments(publishedSegmentsAndCommitMetadata.getSegments(), "Published segments");
publishedSequences.add(sequenceMetadata.getSequenceName());
sequences.remove(sequenceMetadata); sequences.remove(sequenceMetadata);
publishingSequences.remove(sequenceMetadata.getSequenceName()); publishingSequences.remove(sequenceMetadata.getSequenceName());
@ -1157,7 +1160,9 @@ public abstract class SeekableStreamIndexTaskRunner<PartitionIdType, SequenceOff
{ {
for (SequenceMetadata<PartitionIdType, SequenceOffsetType> sequenceMetadata : sequences) { for (SequenceMetadata<PartitionIdType, SequenceOffsetType> sequenceMetadata : sequences) {
sequenceMetadata.updateAssignments(currOffsets, this::isMoreToReadBeforeReadingRecord); sequenceMetadata.updateAssignments(currOffsets, this::isMoreToReadBeforeReadingRecord);
if (!sequenceMetadata.isOpen() && !publishingSequences.contains(sequenceMetadata.getSequenceName())) { if (!sequenceMetadata.isOpen()
&& !publishingSequences.contains(sequenceMetadata.getSequenceName())
&& !publishedSequences.contains(sequenceMetadata.getSequenceName())) {
publishingSequences.add(sequenceMetadata.getSequenceName()); publishingSequences.add(sequenceMetadata.getSequenceName());
try { try {
final Object result = driver.persist(committerSupplier.get()); final Object result = driver.persist(committerSupplier.get());