From 602d95d86591869aa221701c72dfe3f7326da6e3 Mon Sep 17 00:00:00 2001 From: Lucas Capistrant Date: Thu, 16 Jun 2022 09:11:31 -0500 Subject: [PATCH] Add a builder class for TestDruidCoordinatorConfig (#12624) * Add a builder class for TestDruidCoordinatorConfig * updates after review * Fix formatting --- .../CuratorDruidCoordinatorTest.java | 29 +- .../coordinator/DruidCoordinatorTest.java | 59 +-- .../coordinator/HttpLoadQueuePeonTest.java | 34 +- .../server/coordinator/LoadQueuePeonTest.java | 80 +--- .../coordinator/LoadQueuePeonTester.java | 27 +- .../TestDruidCoordinatorConfig.java | 342 +++++++++++++++--- .../coordinator/duty/KillAuditLogTest.java | 112 ++---- .../duty/KillCompactionConfigTest.java | 135 ++----- .../duty/KillDatasourceMetadataTest.java | 140 ++----- .../coordinator/duty/KillRulesTest.java | 112 ++---- .../coordinator/duty/KillSupervisorsTest.java | 112 ++---- .../duty/KillUnusedSegmentsTest.java | 180 +++------ 12 files changed, 549 insertions(+), 813 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 68621ed606a..64ffe8d5fd3 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -165,27 +165,14 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase curator.create().creatingParentsIfNeeded().forPath(DESTINATION_LOAD_PATH); objectMapper = new DefaultObjectMapper(); - druidCoordinatorConfig = new TestDruidCoordinatorConfig( - new Duration(COORDINATOR_START_DELAY), - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT0s"), - false - ); + druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withCoordinatorStartDelay(new Duration(COORDINATOR_START_DELAY)) + .withCoordinatorPeriod(new Duration(COORDINATOR_PERIOD)) + .withCoordinatorKillPeriod(new Duration(COORDINATOR_PERIOD)) + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(new Duration("PT0s")) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); sourceLoadQueueChildrenCache = new PathChildrenCache( curator, SOURCE_LOAD_PATH, diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java index 26975d9aa9d..7b2840d0e21 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java @@ -145,27 +145,13 @@ public class DruidCoordinatorTest extends CuratorTestBase curator.blockUntilConnected(); curator.create().creatingParentsIfNeeded().forPath(LOADPATH); objectMapper = new DefaultObjectMapper(); - druidCoordinatorConfig = new TestDruidCoordinatorConfig( - new Duration(COORDINATOR_START_DELAY), - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT0s"), - false - ); + druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withCoordinatorStartDelay(new Duration(COORDINATOR_START_DELAY)) + .withCoordinatorPeriod(new Duration(COORDINATOR_PERIOD)) + .withCoordinatorKillPeriod(new Duration(COORDINATOR_PERIOD)) + .withLoadQueuePeonRepeatDelay(new Duration("PT0s")) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); pathChildrenCache = new PathChildrenCache( curator, LOADPATH, @@ -925,28 +911,15 @@ public class DruidCoordinatorTest extends CuratorTestBase @Test public void testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments() { - DruidCoordinatorConfig differentConfigUsedInCustomGroup = new TestDruidCoordinatorConfig( - new Duration(COORDINATOR_START_DELAY), - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - new Duration(COORDINATOR_PERIOD), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT0s"), - false, - false - ); + DruidCoordinatorConfig differentConfigUsedInCustomGroup = new TestDruidCoordinatorConfig.Builder() + .withCoordinatorStartDelay(new Duration(COORDINATOR_START_DELAY)) + .withCoordinatorPeriod(new Duration(COORDINATOR_PERIOD)) + .withCoordinatorKillPeriod(new Duration(COORDINATOR_PERIOD)) + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(new Duration("PT0s")) + .withCompactionSkippedLockedIntervals(false) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); CoordinatorCustomDutyGroup compactSegmentCustomGroup = new CoordinatorCustomDutyGroup("group1", Duration.standardSeconds(1), ImmutableList.of(new CompactSegments(differentConfigUsedInCustomGroup, null, null))); CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(compactSegmentCustomGroup)); coordinator = new DruidCoordinator( diff --git a/server/src/test/java/org/apache/druid/server/coordinator/HttpLoadQueuePeonTest.java b/server/src/test/java/org/apache/druid/server/coordinator/HttpLoadQueuePeonTest.java index 019b59a57b2..e270fa1b1da 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/HttpLoadQueuePeonTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/HttpLoadQueuePeonTest.java @@ -73,34 +73,12 @@ public class HttpLoadQueuePeonTest null, null, null, null, 0, 0 ); - final TestDruidCoordinatorConfig config = new TestDruidCoordinatorConfig( - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - Duration.ZERO, - false - ) - { - @Override - public int getHttpLoadQueuePeonBatchSize() - { - return 2; - } - }; + final TestDruidCoordinatorConfig config = new TestDruidCoordinatorConfig.Builder() + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .withHttpLoadQueuePeonBatchSize(2) + .build(); @Test(timeout = 60_000L) public void testSimple() throws Exception diff --git a/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTest.java b/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTest.java index 74af5dc2822..3bb8fcda25a 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTest.java @@ -88,27 +88,11 @@ public class LoadQueuePeonTest extends CuratorTestBase jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), - new TestDruidCoordinatorConfig( - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - Duration.millis(0), - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(Duration.millis(0)) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); loadQueuePeon.start(); @@ -294,27 +278,12 @@ public class LoadQueuePeonTest extends CuratorTestBase Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly - new TestDruidCoordinatorConfig( - null, - null, - null, - null, - new Duration(1), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT1s"), - false - ) + new TestDruidCoordinatorConfig.Builder() + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(new Duration("PT1s")) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); loadQueuePeon.start(); @@ -357,27 +326,12 @@ public class LoadQueuePeonTest extends CuratorTestBase Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly - new TestDruidCoordinatorConfig( - null, - null, - null, - null, - new Duration(1), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT1s"), - false - ) + new TestDruidCoordinatorConfig.Builder() + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(new Duration("PT1s")) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); loadQueuePeon.start(); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTester.java b/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTester.java index e153b3926f5..9abcb105c39 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTester.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/LoadQueuePeonTester.java @@ -37,27 +37,12 @@ public class LoadQueuePeonTester extends CuratorLoadQueuePeon null, Execs.scheduledSingleThreaded("LoadQueuePeonTester--%d"), null, - new TestDruidCoordinatorConfig( - null, - null, - null, - null, - new Duration(1), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - 10, - new Duration("PT1s"), - false - ) + new TestDruidCoordinatorConfig.Builder() + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillMaxSegments(10) + .withLoadQueuePeonRepeatDelay(new Duration("PT1s")) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/TestDruidCoordinatorConfig.java b/server/src/test/java/org/apache/druid/server/coordinator/TestDruidCoordinatorConfig.java index 644f0e187fd..8153dd79428 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/TestDruidCoordinatorConfig.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/TestDruidCoordinatorConfig.java @@ -39,10 +39,15 @@ public class TestDruidCoordinatorConfig extends DruidCoordinatorConfig private final Duration coordinatorRuleKillDurationToRetain; private final Duration coordinatorDatasourceKillPeriod; private final Duration coordinatorDatasourceKillDurationToRetain; - private final Duration getLoadQueuePeonRepeatDelay; + private final Duration loadQueuePeonRepeatDelay; private final int coordinatorKillMaxSegments; private final boolean compactionSkipLockedIntervals; private final boolean coordinatorKillIgnoreDurationToRetain; + private final String loadQueuePeonType; + private final Duration httpLoadQueuePeonRepeatDelay; + private final int curatorLoadQueuePeonNumCallbackThreads; + private final Duration httpLoadQueuePeonHostTimeout; + private final int httpLoadQueuePeonBatchSize; public TestDruidCoordinatorConfig( Duration coordinatorStartDelay, @@ -62,53 +67,14 @@ public class TestDruidCoordinatorConfig extends DruidCoordinatorConfig Duration coordinatorDatasourceKillPeriod, Duration coordinatorDatasourceKillDurationToRetain, int coordinatorKillMaxSegments, - Duration getLoadQueuePeonRepeatDelay, - boolean coordinatorKillIgnoreDurationToRetain - ) - { - this.coordinatorStartDelay = coordinatorStartDelay; - this.coordinatorPeriod = coordinatorPeriod; - this.coordinatorIndexingPeriod = coordinatorIndexingPeriod; - this.metadataStoreManagementPeriod = metadataStoreManagementPeriod; - this.loadTimeoutDelay = loadTimeoutDelay; - this.coordinatorKillPeriod = coordinatorKillPeriod; - this.coordinatorKillDurationToRetain = coordinatorKillDurationToRetain; - this.coordinatorSupervisorKillPeriod = coordinatorSupervisorKillPeriod; - this.coordinatorSupervisorKillDurationToRetain = coordinatorSupervisorKillDurationToRetain; - this.coordinatorAuditKillPeriod = coordinatorAuditKillPeriod; - this.coordinatorAuditKillDurationToRetain = coordinatorAuditKillDurationToRetain; - this.coordinatorCompactionKillPeriod = coordinatorCompactionKillPeriod; - this.coordinatorRuleKillPeriod = coordinatorRuleKillPeriod; - this.coordinatorRuleKillDurationToRetain = coordinatorRuleKillDurationToRetain; - this.coordinatorDatasourceKillPeriod = coordinatorDatasourceKillPeriod; - this.coordinatorDatasourceKillDurationToRetain = coordinatorDatasourceKillDurationToRetain; - this.coordinatorKillMaxSegments = coordinatorKillMaxSegments; - this.getLoadQueuePeonRepeatDelay = getLoadQueuePeonRepeatDelay; - this.compactionSkipLockedIntervals = true; - this.coordinatorKillIgnoreDurationToRetain = coordinatorKillIgnoreDurationToRetain; - } - - public TestDruidCoordinatorConfig( - Duration coordinatorStartDelay, - Duration coordinatorPeriod, - Duration coordinatorIndexingPeriod, - Duration metadataStoreManagementPeriod, - Duration loadTimeoutDelay, - Duration coordinatorKillPeriod, - Duration coordinatorKillDurationToRetain, - Duration coordinatorSupervisorKillPeriod, - Duration coordinatorSupervisorKillDurationToRetain, - Duration coordinatorAuditKillPeriod, - Duration coordinatorAuditKillDurationToRetain, - Duration coordinatorCompactionKillPeriod, - Duration coordinatorRuleKillPeriod, - Duration coordinatorRuleKillDurationToRetain, - Duration coordinatorDatasourceKillPeriod, - Duration coordinatorDatasourceKillDurationToRetain, - int coordinatorKillMaxSegments, - Duration getLoadQueuePeonRepeatDelay, + Duration loadQueuePeonRepeatDelay, boolean compactionSkipLockedIntervals, - boolean coordinatorKillIgnoreDurationToRetain + boolean coordinatorKillIgnoreDurationToRetain, + String loadQueuePeonType, + Duration httpLoadQueuePeonRepeatDelay, + Duration httpLoadQueuePeonHostTimeout, + int httpLoadQueuePeonBatchSize, + int curatorLoadQueuePeonNumCallbackThreads ) { this.coordinatorStartDelay = coordinatorStartDelay; @@ -128,9 +94,14 @@ public class TestDruidCoordinatorConfig extends DruidCoordinatorConfig this.coordinatorDatasourceKillPeriod = coordinatorDatasourceKillPeriod; this.coordinatorDatasourceKillDurationToRetain = coordinatorDatasourceKillDurationToRetain; this.coordinatorKillMaxSegments = coordinatorKillMaxSegments; - this.getLoadQueuePeonRepeatDelay = getLoadQueuePeonRepeatDelay; + this.loadQueuePeonRepeatDelay = loadQueuePeonRepeatDelay; this.compactionSkipLockedIntervals = compactionSkipLockedIntervals; this.coordinatorKillIgnoreDurationToRetain = coordinatorKillIgnoreDurationToRetain; + this.loadQueuePeonType = loadQueuePeonType; + this.httpLoadQueuePeonRepeatDelay = httpLoadQueuePeonRepeatDelay; + this.httpLoadQueuePeonHostTimeout = httpLoadQueuePeonHostTimeout; + this.httpLoadQueuePeonBatchSize = httpLoadQueuePeonBatchSize; + this.curatorLoadQueuePeonNumCallbackThreads = curatorLoadQueuePeonNumCallbackThreads; } @Override @@ -238,7 +209,7 @@ public class TestDruidCoordinatorConfig extends DruidCoordinatorConfig @Override public Duration getLoadQueuePeonRepeatDelay() { - return getLoadQueuePeonRepeatDelay; + return loadQueuePeonRepeatDelay; } @Override @@ -252,4 +223,277 @@ public class TestDruidCoordinatorConfig extends DruidCoordinatorConfig { return coordinatorKillIgnoreDurationToRetain; } + + @Override + public String getLoadQueuePeonType() + { + return loadQueuePeonType; + } + + @Override + public Duration getHttpLoadQueuePeonRepeatDelay() + { + return httpLoadQueuePeonRepeatDelay; + } + + @Override + public int getNumCuratorCallBackThreads() + { + return curatorLoadQueuePeonNumCallbackThreads; + } + + @Override + public Duration getHttpLoadQueuePeonHostTimeout() + { + return httpLoadQueuePeonHostTimeout; + } + + @Override + public int getHttpLoadQueuePeonBatchSize() + { + return httpLoadQueuePeonBatchSize; + } + + public static class Builder + { + private static final Duration DEFAULT_COORDINATOR_START_DELAY = new Duration("PT300s"); + private static final Duration DEFAULT_COORDINATOR_PERIOD = new Duration("PT60s"); + private static final Duration DEFAULT_COORDINATOR_INDEXING_PERIOD = new Duration("PT1800s"); + private static final Duration DEFAULT_METADATA_STORE_MANAGEMENT_PERIOD = new Duration("PT3600s"); + private static final Duration DEFAULT_COORDINATOR_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_KILL_DURATION_TO_RETAION = new Duration("PT7776000s"); + private static final boolean DEFAULT_COORDINATOR_KILL_IGNORE_DURATION_TO_RETAIN = false; + private static final int DEFAULT_COORDINATOR_KILL_MAX_SEGMENTS = 100; + private static final Duration DEFAULT_COORDINATOR_SUPERVISOR_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_SUPERVISOR_KILL_DURATION_TO_RETAIN = new Duration("PT7776000s"); + private static final Duration DEFAULT_COORDINATOR_COMPACTION_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_RULE_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_RULE_KILL_DURATION_TO_RETAIN = new Duration("PT7776000s"); + private static final Duration DEFAULT_COORDINATOR_DATASOURCE_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_DATASOURCE_KILL_DURATION_TO_RETAIN = new Duration("PT7776000s"); + private static final Duration DEFAULT_LOAD_TIMEOUT_DELAY = new Duration(15 * 60 * 1000); + private static final Duration DEFAULT_LOAD_QUEUE_PEON_REPEAT_DELAY = Duration.millis(50); + private static final String DEFAULT_LOAD_QUEUE_PEON_TYPE = "curator"; + private static final int DEFAULT_CURATOR_LOAD_QUEUE_PEON_NUM_CALLBACK_THREADS = 2; + private static final Duration DEFAULT_HTTP_LOAD_QUEUE_PEON_REPEAT_DELAY = Duration.millis(60000); + private static final Duration DEFAULT_HTTP_LOAD_QUEUE_PEON_HOST_TIMEOUT = Duration.millis(300000); + private static final int DEFAULT_HTTP_LOAD_QUEUE_PEON_BATCH_SIZE = 1; + private static final boolean DEFAULT_COMPACTION_SKIP_LOCKED_INTERVALS = true; + private static final Duration DEFAULT_COORDINATOR_AUDIT_KILL_PERIOD = new Duration("PT86400s"); + private static final Duration DEFAULT_COORDINATOR_AUTIT_KILL_DURATION_TO_RETAIN = new Duration("PT7776000s"); + + + private Duration coordinatorStartDelay; + private Duration coordinatorPeriod; + private Duration coordinatorIndexingPeriod; + private Duration metadataStoreManagementPeriod; + private Duration coordinatorKillPeriod; + private Duration coordinatorKillDurationToRetain; + private Boolean coordinatorKillIgnoreDurationToRetain; + private Integer coordinatorKillMaxSegments; + private Duration coordinatorSupervisorKillPeriod; + private Duration coordinatorSupervisorKillDurationToRetain; + private Duration coordinatorCompactionKillPeriod; + private Duration coordinatorRuleKillPeriod; + private Duration coordinatorRuleKillDurationToRetain; + private Duration coordinatorDatasourceKillPeriod; + private Duration coordinatorDatasourceKillDurationToRetain; + private Duration loadTimeoutDelay; + private Duration loadQueuePeonRepeatDelay; + private String loadQueuePeonType; + private Duration httpLoadQueuePeonRepeatDelay; + private Integer curatorLoadQueuePeonNumCallbackThreads; + private Duration httpLoadQueuePeonHostTimeout; + private Integer httpLoadQueuePeonBatchSize; + private Boolean compactionSkippedLockedIntervals; + private Duration coordinatorAuditKillPeriod; + private Duration coordinatorAuditKillDurationToRetain; + + public Builder() + { + } + + public Builder withCoordinatorStartDelay(Duration coordinatorStartDelay) + { + this.coordinatorStartDelay = coordinatorStartDelay; + return this; + } + + public Builder withCoordinatorPeriod(Duration coordinatorPeriod) + { + this.coordinatorPeriod = coordinatorPeriod; + return this; + } + + public Builder withCoordinatorIndexingPeriod(Duration coordinatorIndexingPeriod) + { + this.coordinatorIndexingPeriod = coordinatorIndexingPeriod; + return this; + } + + public Builder withMetadataStoreManagementPeriod(Duration metadataStoreManagementPeriod) + { + this.metadataStoreManagementPeriod = metadataStoreManagementPeriod; + return this; + } + + public Builder withCoordinatorKillPeriod(Duration coordinatorKillPeriod) + { + this.coordinatorKillPeriod = coordinatorKillPeriod; + return this; + } + + public Builder withCoordinatorKillDurationToRetain(Duration coordinatorKillDurationToRetain) + { + this.coordinatorKillDurationToRetain = coordinatorKillDurationToRetain; + return this; + } + + public Builder withCoordinatorKillIgnoreDurationToRetain(boolean coordinatorKillIgnoreDurationToRetain) + { + this.coordinatorKillIgnoreDurationToRetain = coordinatorKillIgnoreDurationToRetain; + return this; + } + + public Builder withCoordinatorKillMaxSegments(int coordinatorKillMaxSegments) + { + this.coordinatorKillMaxSegments = coordinatorKillMaxSegments; + return this; + } + + public Builder withCoordinatorSupervisorKillPeriod(Duration coordinatorSupervisorKillPeriod) + { + this.coordinatorSupervisorKillPeriod = coordinatorSupervisorKillPeriod; + return this; + } + + public Builder withCoordinatorSupervisorKillDurationToRetain(Duration coordinatorSupervisorKillDurationToRetain) + { + this.coordinatorSupervisorKillDurationToRetain = coordinatorSupervisorKillDurationToRetain; + return this; + } + + public Builder withCoordinatorCompactionKillPeriod(Duration coordinatorCompactionKillPeriod) + { + this.coordinatorCompactionKillPeriod = coordinatorCompactionKillPeriod; + return this; + } + + public Builder withCoordinatorRuleKillPeriod(Duration coordinatorRuleKillPeriod) + { + this.coordinatorRuleKillPeriod = coordinatorRuleKillPeriod; + return this; + } + + public Builder withCoordinatorRuleKillDurationToRetain(Duration coordinatorRuleKillDurationToRetain) + { + this.coordinatorRuleKillDurationToRetain = coordinatorRuleKillDurationToRetain; + return this; + } + + public Builder withCoordinatorDatasourceKillPeriod(Duration coordinatorDatasourceKillPeriod) + { + this.coordinatorDatasourceKillPeriod = coordinatorDatasourceKillPeriod; + return this; + } + + public Builder withCoordinatorDatasourceKillDurationToRetain(Duration coordinatorDatasourceKillDurationToRetain) + { + this.coordinatorDatasourceKillDurationToRetain = coordinatorDatasourceKillDurationToRetain; + return this; + } + + public Builder withLoadTimeoutDelay(Duration loadTimeoutDelay) + { + this.loadTimeoutDelay = loadTimeoutDelay; + return this; + } + + public Builder withLoadQueuePeonRepeatDelay(Duration loadQueuePeonRepeatDelay) + { + this.loadQueuePeonRepeatDelay = loadQueuePeonRepeatDelay; + return this; + } + + public Builder withLoadQueuePeonType(String loadQueuePeonType) + { + this.loadQueuePeonType = loadQueuePeonType; + return this; + } + + public Builder withHttpLoadQueuePeonRepeatDelay(Duration httpLoadQueuePeonRepeatDelay) + { + this.httpLoadQueuePeonRepeatDelay = httpLoadQueuePeonRepeatDelay; + return this; + } + + public Builder withCuratorLoadQueuePeonNumCallbackThreads(int curatorLoadQueuePeonNumCallbackThreads) + { + this.curatorLoadQueuePeonNumCallbackThreads = curatorLoadQueuePeonNumCallbackThreads; + return this; + } + + public Builder withHttpLoadQueuePeonHostTimeout(Duration httpLoadQueuePeonHostTimeout) + { + this.httpLoadQueuePeonHostTimeout = httpLoadQueuePeonHostTimeout; + return this; + } + + public Builder withHttpLoadQueuePeonBatchSize(int httpLoadQueuePeonBatchSize) + { + this.httpLoadQueuePeonBatchSize = httpLoadQueuePeonBatchSize; + return this; + } + + public Builder withCompactionSkippedLockedIntervals(boolean compactionSkippedLockedIntervals) + { + this.compactionSkippedLockedIntervals = compactionSkippedLockedIntervals; + return this; + } + + public Builder withCoordianatorAuditKillPeriod(Duration coordinatorAuditKillPeriod) + { + this.coordinatorAuditKillPeriod = coordinatorAuditKillPeriod; + return this; + } + + public Builder withCoordinatorAuditKillDurationToRetain(Duration coordinatorAuditKillDurationToRetain) + { + this.coordinatorAuditKillDurationToRetain = coordinatorAuditKillDurationToRetain; + return this; + } + + public TestDruidCoordinatorConfig build() + { + return new TestDruidCoordinatorConfig( + coordinatorStartDelay == null ? DEFAULT_COORDINATOR_START_DELAY : coordinatorStartDelay, + coordinatorPeriod == null ? DEFAULT_COORDINATOR_PERIOD : coordinatorPeriod, + coordinatorIndexingPeriod == null ? DEFAULT_COORDINATOR_INDEXING_PERIOD : coordinatorIndexingPeriod, + metadataStoreManagementPeriod == null ? DEFAULT_METADATA_STORE_MANAGEMENT_PERIOD : metadataStoreManagementPeriod, + loadTimeoutDelay == null ? DEFAULT_LOAD_TIMEOUT_DELAY : loadTimeoutDelay, + coordinatorKillPeriod == null ? DEFAULT_COORDINATOR_KILL_PERIOD : coordinatorKillPeriod, + coordinatorKillDurationToRetain == null ? DEFAULT_COORDINATOR_KILL_DURATION_TO_RETAION : coordinatorKillDurationToRetain, + coordinatorSupervisorKillPeriod == null ? DEFAULT_COORDINATOR_SUPERVISOR_KILL_PERIOD : coordinatorSupervisorKillPeriod, + coordinatorSupervisorKillDurationToRetain == null ? DEFAULT_COORDINATOR_SUPERVISOR_KILL_DURATION_TO_RETAIN : coordinatorSupervisorKillDurationToRetain, + coordinatorAuditKillPeriod == null ? DEFAULT_COORDINATOR_AUDIT_KILL_PERIOD : coordinatorAuditKillPeriod, + coordinatorAuditKillDurationToRetain == null ? DEFAULT_COORDINATOR_AUTIT_KILL_DURATION_TO_RETAIN : coordinatorAuditKillDurationToRetain, + coordinatorCompactionKillPeriod == null ? DEFAULT_COORDINATOR_COMPACTION_KILL_PERIOD : coordinatorCompactionKillPeriod, + coordinatorRuleKillPeriod == null ? DEFAULT_COORDINATOR_RULE_KILL_PERIOD : coordinatorRuleKillPeriod, + coordinatorRuleKillDurationToRetain == null ? DEFAULT_COORDINATOR_RULE_KILL_DURATION_TO_RETAIN : coordinatorRuleKillDurationToRetain, + coordinatorDatasourceKillPeriod == null ? DEFAULT_COORDINATOR_DATASOURCE_KILL_PERIOD : coordinatorDatasourceKillPeriod, + coordinatorDatasourceKillDurationToRetain == null ? DEFAULT_COORDINATOR_DATASOURCE_KILL_DURATION_TO_RETAIN : coordinatorDatasourceKillDurationToRetain, + coordinatorKillMaxSegments == null ? DEFAULT_COORDINATOR_KILL_MAX_SEGMENTS : coordinatorKillMaxSegments, + loadQueuePeonRepeatDelay == null ? DEFAULT_LOAD_QUEUE_PEON_REPEAT_DELAY : loadQueuePeonRepeatDelay, + compactionSkippedLockedIntervals == null ? DEFAULT_COMPACTION_SKIP_LOCKED_INTERVALS : compactionSkippedLockedIntervals, + coordinatorKillIgnoreDurationToRetain == null ? DEFAULT_COORDINATOR_KILL_IGNORE_DURATION_TO_RETAIN : coordinatorKillIgnoreDurationToRetain, + loadQueuePeonType == null ? DEFAULT_LOAD_QUEUE_PEON_TYPE : loadQueuePeonType, + httpLoadQueuePeonRepeatDelay == null ? DEFAULT_HTTP_LOAD_QUEUE_PEON_REPEAT_DELAY : httpLoadQueuePeonRepeatDelay, + httpLoadQueuePeonHostTimeout == null ? DEFAULT_HTTP_LOAD_QUEUE_PEON_HOST_TIMEOUT : httpLoadQueuePeonHostTimeout, + httpLoadQueuePeonBatchSize == null ? DEFAULT_HTTP_LOAD_QUEUE_PEON_BATCH_SIZE : httpLoadQueuePeonBatchSize, + curatorLoadQueuePeonNumCallbackThreads == null ? DEFAULT_CURATOR_LOAD_QUEUE_PEON_NUM_CALLBACK_THREADS + : curatorLoadQueuePeonNumCallbackThreads + ); + } + + } } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillAuditLogTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillAuditLogTest.java index 84366cde0bb..8480670160a 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillAuditLogTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillAuditLogTest.java @@ -54,27 +54,13 @@ public class KillAuditLogTest @Test public void testRunSkipIfLastRunLessThanPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - new Duration(Long.MAX_VALUE), - new Duration("PT1S"), - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5s")) + .withCoordianatorAuditKillPeriod(new Duration(Long.MAX_VALUE)) + .withCoordinatorAuditKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killAuditLog = new KillAuditLog(mockAuditManager, druidCoordinatorConfig); killAuditLog.run(mockDruidCoordinatorRuntimeParams); Mockito.verifyNoInteractions(mockAuditManager); @@ -84,27 +70,13 @@ public class KillAuditLogTest public void testRunNotSkipIfLastRunMoreThanPeriod() { Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT1S"), - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5s")) + .withCoordianatorAuditKillPeriod(new Duration("PT6S")) + .withCoordinatorAuditKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killAuditLog = new KillAuditLog(mockAuditManager, druidCoordinatorConfig); killAuditLog.run(mockDruidCoordinatorRuntimeParams); Mockito.verify(mockAuditManager).removeAuditLogsOlderThan(ArgumentMatchers.anyLong()); @@ -114,27 +86,13 @@ public class KillAuditLogTest @Test public void testConstructorFailIfInvalidPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - new Duration("PT3S"), - new Duration("PT1S"), - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5s")) + .withCoordianatorAuditKillPeriod(new Duration("PT3S")) + .withCoordinatorAuditKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("coordinator audit kill period must be >= druid.coordinator.period.metadataStoreManagementPeriod"); killAuditLog = new KillAuditLog(mockAuditManager, druidCoordinatorConfig); @@ -143,27 +101,13 @@ public class KillAuditLogTest @Test public void testConstructorFailIfInvalidRetainDuration() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT-1S"), - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordianatorAuditKillPeriod(new Duration("PT6S")) + .withCoordinatorAuditKillDurationToRetain(new Duration("PT-1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("coordinator audit kill retainDuration must be >= 0"); killAuditLog = new KillAuditLog(mockAuditManager, druidCoordinatorConfig); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java index a6e98c80510..916f3ed5e6b 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java @@ -86,27 +86,12 @@ public class KillCompactionConfigTest @Test public void testRunSkipIfLastRunLessThanPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - new Duration(Long.MAX_VALUE), - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorCompactionKillPeriod(new Duration(Long.MAX_VALUE)) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killCompactionConfig = new KillCompactionConfig( druidCoordinatorConfig, mockSqlSegmentsMetadataManager, @@ -123,27 +108,12 @@ public class KillCompactionConfigTest @Test public void testConstructorFailIfInvalidPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - new Duration("PT3S"), - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorCompactionKillPeriod(new Duration("PT3S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Coordinator compaction configuration kill period must be >= druid.coordinator.period.metadataStoreManagementPeriod"); killCompactionConfig = new KillCompactionConfig( @@ -173,27 +143,12 @@ public class KillCompactionConfigTest ArgumentMatchers.eq(CoordinatorCompactionConfig.empty())) ).thenReturn(CoordinatorCompactionConfig.empty()); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorCompactionKillPeriod(new Duration("PT6S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killCompactionConfig = new KillCompactionConfig( druidCoordinatorConfig, mockSqlSegmentsMetadataManager, @@ -279,27 +234,12 @@ public class KillCompactionConfigTest ArgumentMatchers.any()) ).thenReturn(ConfigManager.SetResult.ok()); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorCompactionKillPeriod(new Duration("PT6S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killCompactionConfig = new KillCompactionConfig( druidCoordinatorConfig, mockSqlSegmentsMetadataManager, @@ -399,27 +339,12 @@ public class KillCompactionConfigTest } }); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorCompactionKillPeriod(new Duration("PT6S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killCompactionConfig = new KillCompactionConfig( druidCoordinatorConfig, mockSqlSegmentsMetadataManager, diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillDatasourceMetadataTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillDatasourceMetadataTest.java index 22b3ec07448..9e496033155 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillDatasourceMetadataTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillDatasourceMetadataTest.java @@ -63,27 +63,13 @@ public class KillDatasourceMetadataTest @Test public void testRunSkipIfLastRunLessThanPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - new Duration(Long.MAX_VALUE), - new Duration("PT1S"), - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorDatasourceKillPeriod(new Duration(Long.MAX_VALUE)) + .withCoordinatorDatasourceKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killDatasourceMetadata = new KillDatasourceMetadata(druidCoordinatorConfig, mockIndexerMetadataStorageCoordinator, mockMetadataSupervisorManager); killDatasourceMetadata.run(mockDruidCoordinatorRuntimeParams); Mockito.verifyNoInteractions(mockIndexerMetadataStorageCoordinator); @@ -95,27 +81,13 @@ public class KillDatasourceMetadataTest { Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT1S"), - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorDatasourceKillPeriod(new Duration("PT6S")) + .withCoordinatorDatasourceKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killDatasourceMetadata = new KillDatasourceMetadata(druidCoordinatorConfig, mockIndexerMetadataStorageCoordinator, mockMetadataSupervisorManager); killDatasourceMetadata.run(mockDruidCoordinatorRuntimeParams); Mockito.verify(mockIndexerMetadataStorageCoordinator).removeDataSourceMetadataOlderThan(ArgumentMatchers.anyLong(), ArgumentMatchers.anySet()); @@ -125,27 +97,13 @@ public class KillDatasourceMetadataTest @Test public void testConstructorFailIfInvalidPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT3S"), - new Duration("PT1S"), - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorDatasourceKillPeriod(new Duration("PT3S")) + .withCoordinatorDatasourceKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Coordinator datasource metadata kill period must be >= druid.coordinator.period.metadataStoreManagementPeriod"); killDatasourceMetadata = new KillDatasourceMetadata(druidCoordinatorConfig, mockIndexerMetadataStorageCoordinator, mockMetadataSupervisorManager); @@ -154,27 +112,13 @@ public class KillDatasourceMetadataTest @Test public void testConstructorFailIfInvalidRetainDuration() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT-1S"), - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorDatasourceKillPeriod(new Duration("PT6S")) + .withCoordinatorDatasourceKillDurationToRetain(new Duration("PT-1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Coordinator datasource metadata kill retainDuration must be >= 0"); killDatasourceMetadata = new KillDatasourceMetadata(druidCoordinatorConfig, mockIndexerMetadataStorageCoordinator, mockMetadataSupervisorManager); @@ -185,27 +129,13 @@ public class KillDatasourceMetadataTest { Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT1S"), - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorDatasourceKillPeriod(new Duration("PT6S")) + .withCoordinatorDatasourceKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killDatasourceMetadata = new KillDatasourceMetadata(druidCoordinatorConfig, mockIndexerMetadataStorageCoordinator, mockMetadataSupervisorManager); killDatasourceMetadata.run(mockDruidCoordinatorRuntimeParams); Mockito.verify(mockIndexerMetadataStorageCoordinator).removeDataSourceMetadataOlderThan(ArgumentMatchers.anyLong(), ArgumentMatchers.eq(ImmutableSet.of())); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillRulesTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillRulesTest.java index 0db43018a66..6d98cfe0b58 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillRulesTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillRulesTest.java @@ -61,27 +61,13 @@ public class KillRulesTest @Test public void testRunSkipIfLastRunLessThanPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - new Duration(Long.MAX_VALUE), - new Duration("PT1S"), - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorRuleKillPeriod(new Duration(Long.MAX_VALUE)) + .withCoordinatorRuleKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killRules = new KillRules(druidCoordinatorConfig); killRules.run(mockDruidCoordinatorRuntimeParams); Mockito.verifyNoInteractions(mockRuleManager); @@ -91,27 +77,13 @@ public class KillRulesTest public void testRunNotSkipIfLastRunMoreThanPeriod() { Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT1S"), - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorRuleKillPeriod(new Duration("PT6S")) + .withCoordinatorRuleKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killRules = new KillRules(druidCoordinatorConfig); killRules.run(mockDruidCoordinatorRuntimeParams); Mockito.verify(mockRuleManager).removeRulesForEmptyDatasourcesOlderThan(ArgumentMatchers.anyLong()); @@ -121,27 +93,13 @@ public class KillRulesTest @Test public void testConstructorFailIfInvalidPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT3S"), - new Duration("PT1S"), - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorRuleKillPeriod(new Duration("PT3S")) + .withCoordinatorRuleKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("coordinator rule kill period must be >= druid.coordinator.period.metadataStoreManagementPeriod"); killRules = new KillRules(druidCoordinatorConfig); @@ -150,27 +108,13 @@ public class KillRulesTest @Test public void testConstructorFailIfInvalidRetainDuration() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - null, - null, - null, - null, - null, - new Duration("PT6S"), - new Duration("PT-1S"), - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorRuleKillPeriod(new Duration("PT6S")) + .withCoordinatorRuleKillDurationToRetain(new Duration("PT-1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("coordinator rule kill retainDuration must be >= 0"); killRules = new KillRules(druidCoordinatorConfig); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillSupervisorsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillSupervisorsTest.java index d21b3afb707..0f7c2fff7e2 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillSupervisorsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillSupervisorsTest.java @@ -54,27 +54,13 @@ public class KillSupervisorsTest @Test public void testRunSkipIfLastRunLessThanPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - new Duration(Long.MAX_VALUE), - new Duration("PT1S"), - null, - null, - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorSupervisorKillPeriod(new Duration(Long.MAX_VALUE)) + .withCoordinatorSupervisorKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killSupervisors = new KillSupervisors(druidCoordinatorConfig, mockMetadataSupervisorManager); killSupervisors.run(mockDruidCoordinatorRuntimeParams); Mockito.verifyNoInteractions(mockMetadataSupervisorManager); @@ -84,27 +70,13 @@ public class KillSupervisorsTest public void testRunNotSkipIfLastRunMoreThanPeriod() { Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter); - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - new Duration("PT6S"), - new Duration("PT1S"), - null, - null, - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorSupervisorKillPeriod(new Duration("PT6S")) + .withCoordinatorSupervisorKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); killSupervisors = new KillSupervisors(druidCoordinatorConfig, mockMetadataSupervisorManager); killSupervisors.run(mockDruidCoordinatorRuntimeParams); Mockito.verify(mockMetadataSupervisorManager).removeTerminatedSupervisorsOlderThan(ArgumentMatchers.anyLong()); @@ -114,27 +86,13 @@ public class KillSupervisorsTest @Test public void testConstructorFailIfInvalidPeriod() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - new Duration("PT3S"), - new Duration("PT1S"), - null, - null, - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorSupervisorKillPeriod(new Duration("PT3S")) + .withCoordinatorSupervisorKillDurationToRetain(new Duration("PT1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Coordinator supervisor kill period must be >= druid.coordinator.period.metadataStoreManagementPeriod"); killSupervisors = new KillSupervisors(druidCoordinatorConfig, mockMetadataSupervisorManager); @@ -143,27 +101,13 @@ public class KillSupervisorsTest @Test public void testConstructorFailIfInvalidRetainDuration() { - TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig( - null, - null, - null, - new Duration("PT5S"), - null, - null, - null, - new Duration("PT6S"), - new Duration("PT-1S"), - null, - null, - null, - null, - null, - null, - null, - 10, - null, - false - ); + TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig.Builder() + .withMetadataStoreManagementPeriod(new Duration("PT5S")) + .withCoordinatorSupervisorKillPeriod(new Duration("PT6S")) + .withCoordinatorSupervisorKillDurationToRetain(new Duration("PT-1S")) + .withCoordinatorKillMaxSegments(10) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build(); exception.expect(IllegalArgumentException.class); exception.expectMessage("Coordinator supervisor kill retainDuration must be >= 0"); killSupervisors = new KillSupervisors(druidCoordinatorConfig, mockMetadataSupervisorManager); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index 3c9ebcc3a9f..48d730cd7c7 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -176,27 +176,15 @@ public class KillUnusedSegmentsTest KillUnusedSegments unusedSegmentsKiller = new KillUnusedSegments( segmentsMetadataManager, indexingServiceClient, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); Assert.assertEquals( @@ -217,27 +205,15 @@ public class KillUnusedSegmentsTest KillUnusedSegments unusedSegmentsKiller = new KillUnusedSegments( null, null, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); Assert.assertEquals((Long) Duration.parse("PT86400S").getMillis(), unusedSegmentsKiller.getRetainDuration()); @@ -245,27 +221,15 @@ public class KillUnusedSegmentsTest unusedSegmentsKiller = new KillUnusedSegments( null, null, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT-86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT-86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); Assert.assertEquals((Long) Duration.parse("PT-86400S").getMillis(), unusedSegmentsKiller.getRetainDuration()); } @@ -282,27 +246,15 @@ public class KillUnusedSegmentsTest KillUnusedSegments unusedSegmentsKiller = new KillUnusedSegments( null, null, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - true - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(true) + .build() ); Assert.assertEquals( DateTimes.COMPARE_DATE_AS_STRING_MAX, @@ -313,27 +265,15 @@ public class KillUnusedSegmentsTest unusedSegmentsKiller = new KillUnusedSegments( null, null, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT-86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT-86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); DateTime expectedTime = DateTimes.nowUtc().minus(Duration.parse("PT-86400S").getMillis()); @@ -343,27 +283,15 @@ public class KillUnusedSegmentsTest unusedSegmentsKiller = new KillUnusedSegments( null, null, - new TestDruidCoordinatorConfig( - null, - null, - Duration.parse("PT76400S"), - null, - new Duration(1), - Duration.parse("PT86400S"), - Duration.parse("PT86400S"), - null, - null, - null, - null, - null, - null, - null, - null, - null, - 1000, - Duration.ZERO, - false - ) + new TestDruidCoordinatorConfig.Builder() + .withCoordinatorIndexingPeriod(Duration.parse("PT76400S")) + .withLoadTimeoutDelay(new Duration(1)) + .withCoordinatorKillPeriod(Duration.parse("PT86400S")) + .withCoordinatorKillDurationToRetain(Duration.parse("PT86400S")) + .withCoordinatorKillMaxSegments(1000) + .withLoadQueuePeonRepeatDelay(Duration.ZERO) + .withCoordinatorKillIgnoreDurationToRetain(false) + .build() ); expectedTime = DateTimes.nowUtc().minus(Duration.parse("PT86400S").getMillis()); Assert.assertEquals(expectedTime, unusedSegmentsKiller.getEndTimeUpperLimit());