Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add check for eternity time segment to SqlSegmentsMetadataQuery (#12844) 

* Add check for eternity time segment to SqlSegmentsMetadataQuery

* Add check for half eternities

* Add multiple segments test

* Add failing test to document known issue
This commit is contained in:
Adarsh Sanjeev 2022-08-05 11:03:08 +05:30 committed by GitHub
parent ca4e64aea3
commit 24f8f9e1ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 218 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
View File

@ -233,12 +233,25 @@ public class SqlSegmentsMetadataQuery
)
);
if (i == intervals.size() - 1) {
sb.append(")");
} else {
// Add a special check for a segment which have one end at eternity and the other at some finite value. Since
// we are using string comparison, a segment with this start or end will not be returned otherwise.
if (matchMode.equals(IntervalMode.OVERLAPS)) {
sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" != '%s' AND \"end\" > :start%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i));
sb.append(StringUtils.format(" OR (start != '%s' AND \"end\" = '%s' AND start < :end%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i));
}
if (i != intervals.size() - 1) {
sb.append(" OR ");
}
}
// Add a special check for a single segment with eternity. Since we are using string comparison, a segment with
// this start and end will not be returned otherwise.
// Known Issue: https://github.com/apache/druid/issues/12860
if (matchMode.equals(IntervalMode.OVERLAPS)) {
sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" = '%s')", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd()));
}
sb.append(")");
}
final Query<Map<String, Object>> sql = handle

202
server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
View File

@ -53,6 +53,7 @@ import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@ -97,6 +98,42 @@ public class IndexerSQLMetadataStorageCoordinatorTest
100
);
private final DataSegment eternitySegment = new DataSegment(
"fooDataSource",
Intervals.ETERNITY,
"version",
ImmutableMap.of(),
ImmutableList.of("dim1"),
ImmutableList.of("m1"),
new LinearShardSpec(0),
9,
100
);
private final DataSegment firstHalfEternityRangeSegment = new DataSegment(
"fooDataSource",
new Interval(DateTimes.MIN, DateTimes.of("3000")),
"version",
ImmutableMap.of(),
ImmutableList.of("dim1"),
ImmutableList.of("m1"),
new LinearShardSpec(0),
9,
100
);
private final DataSegment secondHalfEternityRangeSegment = new DataSegment(
"fooDataSource",
new Interval(DateTimes.of("1970"), DateTimes.MAX),
"version",
ImmutableMap.of(),
ImmutableList.of("dim1"),
ImmutableList.of("m1"),
new LinearShardSpec(0),
9,
100
);
private final DataSegment defaultSegment2 = new DataSegment(
"fooDataSource",
Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
@ -254,6 +291,18 @@ public class IndexerSQLMetadataStorageCoordinatorTest
100
);
private final DataSegment hugeTimeRangeSegment4 = new DataSegment(
"hugeTimeRangeDataSource",
Intervals.of("1990-01-01T00Z/19940-01-01T00Z"),
"zversion",
ImmutableMap.of(),
ImmutableList.of("dim1"),
ImmutableList.of("m1"),
new NumberedShardSpec(0, 1),
9,
100
);
private final Set<DataSegment> SEGMENTS = ImmutableSet.of(defaultSegment, defaultSegment2);
private final AtomicLong metadataUpdateCounter = new AtomicLong();
private final AtomicLong segmentTableDropUpdateCounter = new AtomicLong();
@ -1202,6 +1251,159 @@ public class IndexerSQLMetadataStorageCoordinatorTest
);
}
@Test
public void testEternitySegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
eternitySegment
)
);
Assert.assertEquals(
ImmutableSet.of(eternitySegment),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
eternitySegment.getDataSource(),
Intervals.of("2020/2021"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testEternityMultipleSegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
numberedSegment0of0,
eternitySegment
)
);
Assert.assertEquals(
ImmutableSet.of(eternitySegment, numberedSegment0of0),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
eternitySegment.getDataSource(),
Intervals.of("2015/2016"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testFirstHalfEternitySegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
firstHalfEternityRangeSegment
)
);
Assert.assertEquals(
ImmutableSet.of(firstHalfEternityRangeSegment),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
firstHalfEternityRangeSegment.getDataSource(),
Intervals.of("2020/2021"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testFirstHalfEternityMultipleSegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
numberedSegment0of0,
firstHalfEternityRangeSegment
)
);
Assert.assertEquals(
ImmutableSet.of(numberedSegment0of0, firstHalfEternityRangeSegment),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
firstHalfEternityRangeSegment.getDataSource(),
Intervals.of("2015/2016"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testSecondHalfEternitySegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
secondHalfEternityRangeSegment
)
);
Assert.assertEquals(
ImmutableSet.of(secondHalfEternityRangeSegment),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
secondHalfEternityRangeSegment.getDataSource(),
Intervals.of("2020/2021"),
Segments.ONLY_VISIBLE
)
)
);
}
// Known Issue: https://github.com/apache/druid/issues/12860
@Ignore
@Test
public void testLargeIntervalWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
hugeTimeRangeSegment4
)
);
Assert.assertEquals(
ImmutableSet.of(hugeTimeRangeSegment4),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
hugeTimeRangeSegment4.getDataSource(),
Intervals.of("2020/2021"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testSecondHalfEternityMultipleSegmentWithStringComparison() throws IOException
{
coordinator.announceHistoricalSegments(
ImmutableSet.of(
numberedSegment0of0,
secondHalfEternityRangeSegment
)
);
Assert.assertEquals(
ImmutableSet.of(numberedSegment0of0, secondHalfEternityRangeSegment),
ImmutableSet.copyOf(
coordinator.retrieveUsedSegmentsForInterval(
secondHalfEternityRangeSegment.getDataSource(),
Intervals.of("2015/2016"),
Segments.ONLY_VISIBLE
)
)
);
}
@Test
public void testDeleteDataSourceMetadata() throws IOException
{