Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unused coordinator dynamic configs mergeSegmentsLimit, mergeBytesLimit (#17384) 

* Remove unused coordinator dynamic configs

* Update docs and web-console
This commit is contained in:
Kashif Faraz 2024-10-22 09:03:46 +05:30 committed by GitHub
parent 6cf372a7f4
commit 9dfb378711
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 3 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/api-reference/dynamic-configuration-api.md
View File

@ -91,8 +91,6 @@ Host: http://ROUTER_IP:ROUTER_PORT
```json
{
"millisToWaitBeforeDeleting": 900000,
"mergeBytesLimit": 524288000,
"mergeSegmentsLimit": 100,
"maxSegmentsToMove": 100,
"replicantLifetime": 15,
"replicationThrottleLimit": 500,
@ -159,8 +157,6 @@ curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config" \
--header 'Content-Type: application/json' \
--data '{
"millisToWaitBeforeDeleting": 900000,
"mergeBytesLimit": 524288000,
"mergeSegmentsLimit": 100,
"maxSegmentsToMove": 5,
"percentOfSegmentsToConsiderPerMove": 100,
"useBatchedSegmentSampler": true,
@ -192,8 +188,6 @@ Content-Length: 683
{
"millisToWaitBeforeDeleting": 900000,
"mergeBytesLimit": 524288000,
"mergeSegmentsLimit": 100,
"maxSegmentsToMove": 5,
"percentOfSegmentsToConsiderPerMove": 100,
"useBatchedSegmentSampler": true,
@ -295,7 +289,7 @@ Host: http://ROUTER_IP:ROUTER_PORT
"comment": "",
"ip": "127.0.0.1"
},
"payload": "{\"millisToWaitBeforeDeleting\":900000,\"mergeBytesLimit\":524288000,\"mergeSegmentsLimit\":100,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}",
"payload": "{\"millisToWaitBeforeDeleting\":900000,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}",
"auditTime": "2023-10-03T20:59:51.622Z"
}
]

2
docs/configuration/index.md
View File

@ -951,8 +951,6 @@ The following table shows the dynamic configuration properties for the Coordinat
|Property|Description|Default|
|--------|-----------|-------|
|`millisToWaitBeforeDeleting`|How long does the Coordinator need to be a leader before it can start marking overshadowed segments as unused in metadata storage.| 900000 (15 mins)|
|`mergeBytesLimit`|The maximum total uncompressed size in bytes of segments to merge.|524288000L|
|`mergeSegmentsLimit`|The maximum number of segments that can be in a single [append task](../ingestion/tasks.md).|100|
|`smartSegmentLoading`|Enables ["smart" segment loading mode](#smart-segment-loading) which dynamically computes the optimal values of several properties that maximize Coordinator performance.|true|
|`maxSegmentsToMove`|The maximum number of segments that can be moved in a Historical tier at any given time.|100|
|`replicantLifetime`|The maximum number of Coordinator runs for which a segment can wait in the load queue of a Historical before Druid raises an alert.|15|

38
server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
View File

@ -51,8 +51,6 @@ public class CoordinatorDynamicConfig
public static final String CONFIG_KEY = "coordinator.config";
private final long markSegmentAsUnusedDelayMillis;
private final long mergeBytesLimit;
private final int mergeSegmentsLimit;
private final int maxSegmentsToMove;
private final int replicantLifetime;
private final int replicationThrottleLimit;
@ -98,8 +96,6 @@ public class CoordinatorDynamicConfig
// updated to Jackson 2.9 it could be changed, see https://github.com/apache/druid/issues/7152
@JsonProperty("millisToWaitBeforeDeleting")
long markSegmentAsUnusedDelayMillis,
@JsonProperty("mergeBytesLimit") long mergeBytesLimit,
@JsonProperty("mergeSegmentsLimit") int mergeSegmentsLimit,
@JsonProperty("maxSegmentsToMove") int maxSegmentsToMove,
@JsonProperty("replicantLifetime") int replicantLifetime,
@JsonProperty("replicationThrottleLimit") int replicationThrottleLimit,
@ -127,8 +123,6 @@ public class CoordinatorDynamicConfig
{
this.markSegmentAsUnusedDelayMillis =
markSegmentAsUnusedDelayMillis;
this.mergeBytesLimit = mergeBytesLimit;
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.smartSegmentLoading = Builder.valueOrDefault(smartSegmentLoading, Defaults.SMART_SEGMENT_LOADING);
@ -212,18 +206,6 @@ public class CoordinatorDynamicConfig
return markSegmentAsUnusedDelayMillis;
}
@JsonProperty
public long getMergeBytesLimit()
{
return mergeBytesLimit;
}
@JsonProperty
public int getMergeSegmentsLimit()
{
return mergeSegmentsLimit;
}
@JsonProperty
public int getMaxSegmentsToMove()
{
@ -294,7 +276,7 @@ public class CoordinatorDynamicConfig
* List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning'
* servers, and segments will be moved away from them to be placed on non-decommissioning servers.
*
* @return list of host:port entries
* @return Set of host:port entries
*/
@JsonProperty
public Set<String> getDecommissioningNodes()
@ -332,8 +314,6 @@ public class CoordinatorDynamicConfig
return "CoordinatorDynamicConfig{" +
"leadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments="
+ markSegmentAsUnusedDelayMillis +
", mergeBytesLimit=" + mergeBytesLimit +
", mergeSegmentsLimit=" + mergeSegmentsLimit +
", maxSegmentsToMove=" + maxSegmentsToMove +
", replicantLifetime=" + replicantLifetime +
", replicationThrottleLimit=" + replicationThrottleLimit +
@ -362,8 +342,6 @@ public class CoordinatorDynamicConfig
CoordinatorDynamicConfig that = (CoordinatorDynamicConfig) o;
return markSegmentAsUnusedDelayMillis == that.markSegmentAsUnusedDelayMillis
&& mergeBytesLimit == that.mergeBytesLimit
&& mergeSegmentsLimit == that.mergeSegmentsLimit
&& maxSegmentsToMove == that.maxSegmentsToMove
&& balancerComputeThreads == that.balancerComputeThreads
&& replicantLifetime == that.replicantLifetime
@ -389,8 +367,6 @@ public class CoordinatorDynamicConfig
{
return Objects.hash(
markSegmentAsUnusedDelayMillis,
mergeBytesLimit,
mergeSegmentsLimit,
maxSegmentsToMove,
replicantLifetime,
replicationThrottleLimit,
@ -423,8 +399,6 @@ public class CoordinatorDynamicConfig
private static class Defaults
{
static final long LEADING_MILLIS_BEFORE_MARK_UNUSED = TimeUnit.MINUTES.toMillis(15);
static final long MERGE_BYTES_LIMIT = 524_288_000L;
static final int MERGE_SEGMENTS_LIMIT = 100;
static final int MAX_SEGMENTS_TO_MOVE = 100;
static final int REPLICANT_LIFETIME = 15;
static final int REPLICATION_THROTTLE_LIMIT = 500;
@ -441,8 +415,6 @@ public class CoordinatorDynamicConfig
public static class Builder
{
private Long markSegmentAsUnusedDelayMillis;
private Long mergeBytesLimit;
private Integer mergeSegmentsLimit;
private Integer maxSegmentsToMove;
private Integer replicantLifetime;
private Integer replicationThrottleLimit;
@ -466,8 +438,6 @@ public class CoordinatorDynamicConfig
@JsonCreator
public Builder(
@JsonProperty("millisToWaitBeforeDeleting") @Nullable Long markSegmentAsUnusedDelayMillis,
@JsonProperty("mergeBytesLimit") @Nullable Long mergeBytesLimit,
@JsonProperty("mergeSegmentsLimit") @Nullable Integer mergeSegmentsLimit,
@JsonProperty("maxSegmentsToMove") @Nullable Integer maxSegmentsToMove,
@JsonProperty("replicantLifetime") @Nullable Integer replicantLifetime,
@JsonProperty("replicationThrottleLimit") @Nullable Integer replicationThrottleLimit,
@ -486,8 +456,6 @@ public class CoordinatorDynamicConfig
)
{
this.markSegmentAsUnusedDelayMillis = markSegmentAsUnusedDelayMillis;
this.mergeBytesLimit = mergeBytesLimit;
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.replicantLifetime = replicantLifetime;
this.replicationThrottleLimit = replicationThrottleLimit;
@ -600,8 +568,6 @@ public class CoordinatorDynamicConfig
markSegmentAsUnusedDelayMillis,
Defaults.LEADING_MILLIS_BEFORE_MARK_UNUSED
),
valueOrDefault(mergeBytesLimit, Defaults.MERGE_BYTES_LIMIT),
valueOrDefault(mergeSegmentsLimit, Defaults.MERGE_SEGMENTS_LIMIT),
valueOrDefault(maxSegmentsToMove, Defaults.MAX_SEGMENTS_TO_MOVE),
valueOrDefault(replicantLifetime, Defaults.REPLICANT_LIFETIME),
valueOrDefault(replicationThrottleLimit, Defaults.REPLICATION_THROTTLE_LIMIT),
@ -632,8 +598,6 @@ public class CoordinatorDynamicConfig
markSegmentAsUnusedDelayMillis,
defaults.getMarkSegmentAsUnusedDelayMillis()
),
valueOrDefault(mergeBytesLimit, defaults.getMergeBytesLimit()),
valueOrDefault(mergeSegmentsLimit, defaults.getMergeSegmentsLimit()),
valueOrDefault(maxSegmentsToMove, defaults.getMaxSegmentsToMove()),
valueOrDefault(replicantLifetime, defaults.getReplicantLifetime()),
valueOrDefault(replicationThrottleLimit, defaults.getReplicationThrottleLimit()),

88
server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java
View File

@ -41,8 +41,6 @@ public class CoordinatorDynamicConfigTest
{
String jsonStr = "{\n"
+ " \"millisToWaitBeforeDeleting\": 1,\n"
+ " \"mergeBytesLimit\": 1,\n"
+ " \"mergeSegmentsLimit\" : 1,\n"
+ " \"maxSegmentsToMove\": 1,\n"
+ " \"replicantLifetime\": 1,\n"
+ " \"replicationThrottleLimit\": 1,\n"
@ -73,13 +71,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
decommissioning,
false,
@ -93,13 +88,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
ImmutableSet.of("host1"),
false,
@ -113,13 +105,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
ImmutableSet.of("host1"),
false,
@ -133,13 +122,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
ImmutableSet.of("host1"),
true,
@ -153,13 +139,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
ImmutableSet.of("host1"),
true,
@ -173,13 +156,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.15,
2,
false,
1,
ImmutableSet.of("host1"),
true,
@ -193,13 +173,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
2,
false,
1,
ImmutableSet.of("host1"),
true,
@ -213,13 +190,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
5,
false,
1,
ImmutableSet.of("host1"),
true,
@ -244,8 +218,6 @@ public class CoordinatorDynamicConfigTest
public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources()
{
CoordinatorDynamicConfig config = new CoordinatorDynamicConfig(
1,
1,
1,
1,
1,
@ -270,8 +242,6 @@ public class CoordinatorDynamicConfigTest
public void testConstructorWithSpecificDataSourcesToKillShouldNotKillUnusedSegmentsInAllDatasources()
{
CoordinatorDynamicConfig config = new CoordinatorDynamicConfig(
1,
1,
1,
1,
1,
@ -297,8 +267,6 @@ public class CoordinatorDynamicConfigTest
{
String jsonStr = "{\n"
+ " \"millisToWaitBeforeDeleting\": 1,\n"
+ " \"mergeBytesLimit\": 1,\n"
+ " \"mergeSegmentsLimit\" : 1,\n"
+ " \"maxSegmentsToMove\": 1,\n"
+ " \"replicantLifetime\": 1,\n"
+ " \"replicationThrottleLimit\": 1,\n"
@ -324,13 +292,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
Integer.MAX_VALUE,
false,
1,
decommissioning,
false,
@ -344,13 +309,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
Integer.MAX_VALUE,
false,
1,
ImmutableSet.of("host1"),
false,
@ -364,13 +326,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
Integer.MAX_VALUE,
false,
1,
ImmutableSet.of("host1"),
false,
@ -383,8 +342,6 @@ public class CoordinatorDynamicConfigTest
{
String jsonStr = "{\n"
+ " \"millisToWaitBeforeDeleting\": 1,\n"
+ " \"mergeBytesLimit\": 1,\n"
+ " \"mergeSegmentsLimit\" : 1,\n"
+ " \"maxSegmentsToMove\": 1,\n"
+ " \"replicantLifetime\": 1,\n"
+ " \"replicationThrottleLimit\": 1,\n"
@ -408,13 +365,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
ImmutableSet.of("test1", "test2"),
0.1,
Integer.MAX_VALUE,
false,
1,
ImmutableSet.of(),
false,
@ -427,8 +381,6 @@ public class CoordinatorDynamicConfigTest
{
String jsonStr = "{\n"
+ " \"millisToWaitBeforeDeleting\": 1,\n"
+ " \"mergeBytesLimit\": 1,\n"
+ " \"mergeSegmentsLimit\" : 1,\n"
+ " \"maxSegmentsToMove\": 1,\n"
+ " \"replicantLifetime\": 1,\n"
+ " \"replicationThrottleLimit\": 1,\n"
@ -452,13 +404,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
whitelist,
0.1,
Integer.MAX_VALUE,
false,
1,
decommissioning,
false,
@ -471,8 +420,6 @@ public class CoordinatorDynamicConfigTest
{
String jsonStr = "{\n"
+ " \"millisToWaitBeforeDeleting\": 1,\n"
+ " \"mergeBytesLimit\": 1,\n"
+ " \"mergeSegmentsLimit\" : 1,\n"
+ " \"maxSegmentsToMove\": 1,\n"
+ " \"replicantLifetime\": 1,\n"
+ " \"replicationThrottleLimit\": 1,\n"
@ -492,13 +439,10 @@ public class CoordinatorDynamicConfigTest
1,
1,
1,
1,
1,
2,
ImmutableSet.of(),
0.1,
Integer.MAX_VALUE,
true,
EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
ImmutableSet.of(),
false,
@ -514,8 +458,6 @@ public class CoordinatorDynamicConfigTest
assertConfig(
defaultConfig,
900000,
524288000,
100,
100,
15,
500,
@ -523,7 +465,6 @@ public class CoordinatorDynamicConfigTest
emptyList,
0.1,
Integer.MAX_VALUE,
true,
EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
emptyList,
false,
@ -542,8 +483,6 @@ public class CoordinatorDynamicConfigTest
assertConfig(
config,
900000,
524288000,
100,
100,
15,
500,
@ -551,7 +490,6 @@ public class CoordinatorDynamicConfigTest
ImmutableSet.of("DATASOURCE"),
0.1,
Integer.MAX_VALUE,
false,
EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
ImmutableSet.of(),
false,
@ -569,26 +507,7 @@ public class CoordinatorDynamicConfigTest
Assert.assertEquals(
current,
new CoordinatorDynamicConfig.Builder(
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
).build(current)
CoordinatorDynamicConfig.builder().build(current)
);
}
@ -604,8 +523,6 @@ public class CoordinatorDynamicConfigTest
private void assertConfig(
CoordinatorDynamicConfig config,
long expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments,
long expectedMergeBytesLimit,
int expectedMergeSegmentsLimit,
int expectedMaxSegmentsToMove,
int expectedReplicantLifetime,
int expectedReplicationThrottleLimit,
@ -613,7 +530,6 @@ public class CoordinatorDynamicConfigTest
Set<String> expectedSpecificDataSourcesToKillUnusedSegmentsIn,
Double expectedKillTaskSlotRatio,
@Nullable Integer expectedMaxKillTaskSlots,
boolean expectedKillUnusedSegmentsInAllDataSources,
int expectedMaxSegmentsInNodeLoadingQueue,
Set<String> decommissioningNodes,
boolean pauseCoordination,
@ -624,8 +540,6 @@ public class CoordinatorDynamicConfigTest
expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments,
config.getMarkSegmentAsUnusedDelayMillis()
);
Assert.assertEquals(expectedMergeBytesLimit, config.getMergeBytesLimit());
Assert.assertEquals(expectedMergeSegmentsLimit, config.getMergeSegmentsLimit());
Assert.assertEquals(expectedMaxSegmentsToMove, config.getMaxSegmentsToMove());
Assert.assertEquals(expectedReplicantLifetime, config.getReplicantLifetime());
Assert.assertEquals(expectedReplicationThrottleLimit, config.getReplicationThrottleLimit());

2
web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
View File

@ -20,8 +20,6 @@ import type { CoordinatorDynamicConfig } from './coordinator-dynamic-config';
export const DEFAULT_COORDINATOR_DYNAMIC_CONFIG: CoordinatorDynamicConfig = {
millisToWaitBeforeDeleting: 900000,
mergeBytesLimit: 524288000,
mergeSegmentsLimit: 100,
maxSegmentsToMove: 100,
replicantLifetime: 15,
replicationThrottleLimit: 500,

14
web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
View File

@ -30,8 +30,6 @@ export interface CoordinatorDynamicConfig {
maxKillTaskSlots?: number;
killPendingSegmentsSkipList?: string[];
maxSegmentsInNodeLoadingQueue?: number;
mergeBytesLimit?: number;
mergeSegmentsLimit?: number;
millisToWaitBeforeDeleting?: number;
replicantLifetime?: number;
replicationThrottleLimit?: number;
@ -220,18 +218,6 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field<CoordinatorDynamicConfig>[
</>
),
},
{
name: 'mergeBytesLimit',
type: 'size-bytes',
defaultValue: 524288000,
info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
},
{
name: 'mergeSegmentsLimit',
type: 'number',
defaultValue: 100,
info: <>The maximum number of segments that can be in a single append task.</>,
},
{
name: 'millisToWaitBeforeDeleting',
type: 'number',