Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Do not allocate week granular segments unless requested (#15589) 

* Do not allocate week granular segments unless explicitly requested
This commit is contained in:
AmatyaAvadhanula 2024-01-05 12:14:52 +05:30 committed by GitHub
parent d8830b64fc
commit c41e99e10c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 102 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/quickstart/tutorial/rollup-index.json
View File

@ -20,7 +20,7 @@
],
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "week",
"segmentGranularity" : "day",
"queryGranularity" : "minute",
"intervals" : ["2018-01-01/2018-01-03"],
"rollup" : true

2
examples/quickstart/tutorial/transform-index.json
View File

@ -20,7 +20,7 @@
],
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "week",
"segmentGranularity" : "day",
"queryGranularity" : "minute",
"intervals" : ["2018-01-01/2018-01-03"],
"rollup" : true

2
examples/quickstart/tutorial/updates-append-index.json
View File

@ -18,7 +18,7 @@
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "week",
"segmentGranularity": "day",
"queryGranularity": "minute",
"intervals": ["2018-01-01/2018-01-03"],
"rollup": true

2
examples/quickstart/tutorial/updates-append-index2.json
View File

@ -18,7 +18,7 @@
],
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "week",
"segmentGranularity" : "day",
"queryGranularity" : "minute",
"intervals" : ["2018-01-01/2018-01-03"],
"rollup" : true

2
examples/quickstart/tutorial/updates-init-index.json
View File

@ -18,7 +18,7 @@
],
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "week",
"segmentGranularity" : "day",
"queryGranularity" : "minute",
"intervals" : ["2018-01-01/2018-01-03"],
"rollup" : true

2
examples/quickstart/tutorial/updates-overwrite-index.json
View File

@ -18,7 +18,7 @@
],
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "week",
"segmentGranularity" : "day",
"queryGranularity" : "minute",
"intervals" : ["2018-01-01/2018-01-03"],
"rollup" : true

12
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentAllocateAction.java
View File

@ -52,9 +52,13 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
/**
* Allocates a pending segment for a given timestamp. The preferredSegmentGranularity is used if there are no prior
* segments for the given timestamp, or if the prior segments for the given timestamp are already at the
* preferredSegmentGranularity. Otherwise, the prior segments will take precedence.
* Allocates a pending segment for a given timestamp.
* If a visible chunk of used segments contains the interval with the query granularity containing the timestamp,
* the pending segment is allocated with its interval.
* Else, if the interval with the preferred segment granularity containing the timestamp has no overlap
* with the existing used segments, the preferred segment granularity is used.
* Else, find the coarsest segment granularity, containing the interval with the query granularity for the timestamp,
* that does not overlap with the existing used segments. This granularity is used for allocation if it exists.
* <p/>
* This action implicitly acquires some task locks when it allocates segments. You do not have to acquire them
* beforehand, although you *do* have to release them yourself. (Note that task locks are automatically released when
@ -62,6 +66,8 @@ import java.util.stream.Collectors;
* <p/>
* If this action cannot acquire an appropriate task lock, or if it cannot expand an existing segment set, it returns
* null.
* </p>
* Do NOT allocate WEEK granularity segments unless the preferred segment granularity is WEEK.
*/
public class SegmentAllocateAction implements TaskAction<SegmentIdWithShardSpec>
{

61
indexing-service/src/test/java/org/apache/druid/indexing/common/actions/SegmentAllocateActionTest.java
View File

@ -59,6 +59,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -995,6 +996,66 @@ public class SegmentAllocateActionTest
Assert.assertEquals(Intervals.ETERNITY, id2.getInterval());
}
@Test
public void testAllocateWeekOnlyWhenWeekIsPreferred()
{
final Task task = NoopTask.create();
taskActionTestKit.getTaskLockbox().add(task);
final SegmentIdWithShardSpec id1 = allocate(
task,
DateTimes.of("2023-12-16"),
Granularities.MINUTE,
Granularities.HOUR,
"s1",
null
);
final SegmentIdWithShardSpec id2 = allocate(
task,
DateTimes.of("2023-12-18"),
Granularities.MINUTE,
Granularities.WEEK,
"s2",
null
);
Assert.assertNotNull(id1);
Assert.assertNotNull(id2);
Assert.assertEquals(Duration.ofHours(1).toMillis(), id1.getInterval().toDurationMillis());
Assert.assertEquals(Duration.ofDays(7).toMillis(), id2.getInterval().toDurationMillis());
}
@Test
public void testAllocateDayWhenMonthNotPossible()
{
final Task task = NoopTask.create();
taskActionTestKit.getTaskLockbox().add(task);
final SegmentIdWithShardSpec id1 = allocate(
task,
DateTimes.of("2023-12-16"),
Granularities.MINUTE,
Granularities.HOUR,
"s1",
null
);
final SegmentIdWithShardSpec id2 = allocate(
task,
DateTimes.of("2023-12-18"),
Granularities.MINUTE,
Granularities.MONTH,
"s2",
null
);
Assert.assertNotNull(id1);
Assert.assertNotNull(id2);
Assert.assertEquals(Duration.ofHours(1).toMillis(), id1.getInterval().toDurationMillis());
Assert.assertEquals(Duration.ofDays(1).toMillis(), id2.getInterval().toDurationMillis());
}
private SegmentIdWithShardSpec allocate(
final Task task,
final DateTime timestamp,

5
processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularity.java
View File

@ -106,6 +106,8 @@ public abstract class Granularity implements Cacheable
* ALL will not be returned unless the provided granularity is ALL. NONE will never be returned, even if the
* provided granularity is NONE. This is because the main usage of this function in production is segment
* allocation, and we do not wish to generate NONE-granular segments.
*
* The list of granularities returned contains WEEK only if the requested granularity is WEEK.
*/
public static List<Granularity> granularitiesFinerThan(final Granularity gran0)
{
@ -117,6 +119,9 @@ public abstract class Granularity implements Cacheable
if ((gran == GranularityType.ALL && !gran0.equals(Granularities.ALL)) || gran == GranularityType.NONE) {
continue;
}
if (gran == GranularityType.WEEK && !gran0.equals(Granularities.WEEK)) {
continue;
}
final Granularity segmentGranularity = gran.create(origin, tz);
final long segmentGranularityDurationMillis = segmentGranularity.bucket(DateTimes.EPOCH).toDurationMillis();
final long gran0DurationMillis = gran0.bucket(DateTimes.EPOCH).toDurationMillis();

22
processing/src/test/java/org/apache/druid/java/util/common/GranularityTest.java
View File

@ -1002,6 +1002,27 @@ public class GranularityTest
);
}
@Test
public void testGranularitiesFinerThanWeek()
{
Assert.assertEquals(
ImmutableList.of(
Granularities.WEEK,
Granularities.DAY,
Granularities.EIGHT_HOUR,
Granularities.SIX_HOUR,
Granularities.HOUR,
Granularities.THIRTY_MINUTE,
Granularities.FIFTEEN_MINUTE,
Granularities.TEN_MINUTE,
Granularities.FIVE_MINUTE,
Granularities.MINUTE,
Granularities.SECOND
),
Granularity.granularitiesFinerThan(Granularities.WEEK)
);
}
@Test
public void testGranularitiesFinerThanAll()
{
@ -1011,7 +1032,6 @@ public class GranularityTest
Granularities.YEAR,
Granularities.QUARTER,
Granularities.MONTH,
Granularities.WEEK,
Granularities.DAY,
Granularities.EIGHT_HOUR,
Granularities.SIX_HOUR,