diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java index 4ecf7bebe6d..0c3ddfc8905 100644 --- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java +++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java @@ -233,12 +233,25 @@ public class SqlSegmentsMetadataQuery ) ); - if (i == intervals.size() - 1) { - sb.append(")"); - } else { + // Add a special check for a segment which have one end at eternity and the other at some finite value. Since + // we are using string comparison, a segment with this start or end will not be returned otherwise. + if (matchMode.equals(IntervalMode.OVERLAPS)) { + sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" != '%s' AND \"end\" > :start%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i)); + sb.append(StringUtils.format(" OR (start != '%s' AND \"end\" = '%s' AND start < :end%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i)); + } + + if (i != intervals.size() - 1) { sb.append(" OR "); } } + + // Add a special check for a single segment with eternity. Since we are using string comparison, a segment with + // this start and end will not be returned otherwise. + // Known Issue: https://github.com/apache/druid/issues/12860 + if (matchMode.equals(IntervalMode.OVERLAPS)) { + sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" = '%s')", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd())); + } + sb.append(")"); } final Query> sql = handle diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index fca3366b886..6f8d0caaf8a 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -53,6 +53,7 @@ import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -97,6 +98,42 @@ public class IndexerSQLMetadataStorageCoordinatorTest 100 ); + private final DataSegment eternitySegment = new DataSegment( + "fooDataSource", + Intervals.ETERNITY, + "version", + ImmutableMap.of(), + ImmutableList.of("dim1"), + ImmutableList.of("m1"), + new LinearShardSpec(0), + 9, + 100 + ); + + + private final DataSegment firstHalfEternityRangeSegment = new DataSegment( + "fooDataSource", + new Interval(DateTimes.MIN, DateTimes.of("3000")), + "version", + ImmutableMap.of(), + ImmutableList.of("dim1"), + ImmutableList.of("m1"), + new LinearShardSpec(0), + 9, + 100 + ); + + private final DataSegment secondHalfEternityRangeSegment = new DataSegment( + "fooDataSource", + new Interval(DateTimes.of("1970"), DateTimes.MAX), + "version", + ImmutableMap.of(), + ImmutableList.of("dim1"), + ImmutableList.of("m1"), + new LinearShardSpec(0), + 9, + 100 + ); private final DataSegment defaultSegment2 = new DataSegment( "fooDataSource", Intervals.of("2015-01-01T00Z/2015-01-02T00Z"), @@ -254,6 +291,18 @@ public class IndexerSQLMetadataStorageCoordinatorTest 100 ); + private final DataSegment hugeTimeRangeSegment4 = new DataSegment( + "hugeTimeRangeDataSource", + Intervals.of("1990-01-01T00Z/19940-01-01T00Z"), + "zversion", + ImmutableMap.of(), + ImmutableList.of("dim1"), + ImmutableList.of("m1"), + new NumberedShardSpec(0, 1), + 9, + 100 + ); + private final Set SEGMENTS = ImmutableSet.of(defaultSegment, defaultSegment2); private final AtomicLong metadataUpdateCounter = new AtomicLong(); private final AtomicLong segmentTableDropUpdateCounter = new AtomicLong(); @@ -1202,6 +1251,159 @@ public class IndexerSQLMetadataStorageCoordinatorTest ); } + + @Test + public void testEternitySegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + eternitySegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(eternitySegment), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + eternitySegment.getDataSource(), + Intervals.of("2020/2021"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + @Test + public void testEternityMultipleSegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + numberedSegment0of0, + eternitySegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(eternitySegment, numberedSegment0of0), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + eternitySegment.getDataSource(), + Intervals.of("2015/2016"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + @Test + public void testFirstHalfEternitySegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + firstHalfEternityRangeSegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(firstHalfEternityRangeSegment), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + firstHalfEternityRangeSegment.getDataSource(), + Intervals.of("2020/2021"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + @Test + public void testFirstHalfEternityMultipleSegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + numberedSegment0of0, + firstHalfEternityRangeSegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(numberedSegment0of0, firstHalfEternityRangeSegment), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + firstHalfEternityRangeSegment.getDataSource(), + Intervals.of("2015/2016"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + @Test + public void testSecondHalfEternitySegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + secondHalfEternityRangeSegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(secondHalfEternityRangeSegment), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + secondHalfEternityRangeSegment.getDataSource(), + Intervals.of("2020/2021"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + // Known Issue: https://github.com/apache/druid/issues/12860 + @Ignore + @Test + public void testLargeIntervalWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + hugeTimeRangeSegment4 + ) + ); + + Assert.assertEquals( + ImmutableSet.of(hugeTimeRangeSegment4), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + hugeTimeRangeSegment4.getDataSource(), + Intervals.of("2020/2021"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + + @Test + public void testSecondHalfEternityMultipleSegmentWithStringComparison() throws IOException + { + coordinator.announceHistoricalSegments( + ImmutableSet.of( + numberedSegment0of0, + secondHalfEternityRangeSegment + ) + ); + + Assert.assertEquals( + ImmutableSet.of(numberedSegment0of0, secondHalfEternityRangeSegment), + ImmutableSet.copyOf( + coordinator.retrieveUsedSegmentsForInterval( + secondHalfEternityRangeSegment.getDataSource(), + Intervals.of("2015/2016"), + Segments.ONLY_VISIBLE + ) + ) + ); + } + @Test public void testDeleteDataSourceMetadata() throws IOException {