Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix usage of maxColumnsToMerge in auto-compaction tuning config (#12551) 

Issue: 
Even though `CompactionTuningConfig` allows a `maxColumnsToMerge` config
(to optimize memory usage, particulary for datasources with many dimensions),
the corresponding client object `ClientCompactionTaskQueryTuningConfig`
(used by the coordinator duty `CompactSegments` to trigger auto-compaction)
does not contain this field. Thus, the value of `maxColumnsToMerge` specified
in any datasource compaction config is ignored.

Changes:
- Add field `maxColumnsToMerge` in `ClientCompactionTaskQueryTuningConfig`
  and `UserCompactionTaskQueryTuningConfig`
- Fix tests
This commit is contained in:
superivaj 2022-05-20 18:53:08 +02:00 committed by GitHub
parent 69aac6c8dd
commit f9bdb3b236
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 68 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java
View File

@ -122,7 +122,8 @@ public class ClientCompactionTaskQuerySerdeTest
new Duration(3000L),
7,
1000,
100
100,
2
),
new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true),
new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))),
@ -287,7 +288,7 @@ public class ClientCompactionTaskQuerySerdeTest
null,
null,
null,
null,
2,
null,
null
)
@ -342,7 +343,8 @@ public class ClientCompactionTaskQuerySerdeTest
new Duration(3000L),
7,
1000,
100
100,
2
),
new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true),
new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))),

3
integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java
View File

@ -67,7 +67,8 @@ public class CompactionUtil
null,
null,
null,
1
1,
null
),
null,
null,

3
integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
View File

@ -1632,7 +1632,8 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
null,
null,
null,
1
1,
null
),
granularitySpec,
dimensionsSpec,

3
integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java
View File

@ -95,7 +95,8 @@ public class ITAutoCompactionUpgradeTest extends AbstractIndexerTest
null,
null,
null,
1
1,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
null,

20
server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java
View File

@ -75,6 +75,8 @@ public class ClientCompactionTaskQueryTuningConfig
@Nullable
private final Integer totalNumMergeTasks;
@Nullable
private final Integer maxColumnsToMerge;
@Nullable
private final AppendableIndexSpec appendableIndexSpec;
public static ClientCompactionTaskQueryTuningConfig from(
@ -103,6 +105,7 @@ public class ClientCompactionTaskQueryTuningConfig
null,
null,
null,
null,
null
);
} else {
@ -128,7 +131,8 @@ public class ClientCompactionTaskQueryTuningConfig
userCompactionTaskQueryTuningConfig.getChatHandlerTimeout(),
userCompactionTaskQueryTuningConfig.getChatHandlerNumRetries(),
userCompactionTaskQueryTuningConfig.getMaxNumSegmentsToMerge(),
userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks()
userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks(),
userCompactionTaskQueryTuningConfig.getMaxColumnsToMerge()
);
}
}
@ -153,7 +157,8 @@ public class ClientCompactionTaskQueryTuningConfig
@JsonProperty("chatHandlerTimeout") @Nullable Duration chatHandlerTimeout,
@JsonProperty("chatHandlerNumRetries") @Nullable Integer chatHandlerNumRetries,
@JsonProperty("maxNumSegmentsToMerge") @Nullable Integer maxNumSegmentsToMerge,
@JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
@JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
this.maxRowsPerSegment = maxRowsPerSegment;
@ -175,6 +180,7 @@ public class ClientCompactionTaskQueryTuningConfig
this.chatHandlerNumRetries = chatHandlerNumRetries;
this.maxNumSegmentsToMerge = maxNumSegmentsToMerge;
this.totalNumMergeTasks = totalNumMergeTasks;
this.maxColumnsToMerge = maxColumnsToMerge;
}
@JsonProperty
@ -318,6 +324,13 @@ public class ClientCompactionTaskQueryTuningConfig
return totalNumMergeTasks;
}
@JsonProperty
@Nullable
public Integer getMaxColumnsToMerge()
{
return maxColumnsToMerge;
}
@JsonProperty
@Nullable
public AppendableIndexSpec getAppendableIndexSpec()
@ -353,6 +366,7 @@ public class ClientCompactionTaskQueryTuningConfig
Objects.equals(chatHandlerNumRetries, that.chatHandlerNumRetries) &&
Objects.equals(maxNumSegmentsToMerge, that.maxNumSegmentsToMerge) &&
Objects.equals(totalNumMergeTasks, that.totalNumMergeTasks) &&
Objects.equals(maxColumnsToMerge, that.maxColumnsToMerge) &&
Objects.equals(appendableIndexSpec, that.appendableIndexSpec);
}
@ -378,6 +392,7 @@ public class ClientCompactionTaskQueryTuningConfig
chatHandlerNumRetries,
maxNumSegmentsToMerge,
totalNumMergeTasks,
maxColumnsToMerge,
appendableIndexSpec
);
}
@ -404,6 +419,7 @@ public class ClientCompactionTaskQueryTuningConfig
", chatHandlerNumRetries=" + chatHandlerNumRetries +
", maxNumSegmentsToMerge=" + maxNumSegmentsToMerge +
", totalNumMergeTasks=" + totalNumMergeTasks +
", maxColumnsToMerge=" + maxColumnsToMerge +
", appendableIndexSpec=" + appendableIndexSpec +
'}';
}

6
server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java
View File

@ -53,7 +53,8 @@ public class UserCompactionTaskQueryTuningConfig extends ClientCompactionTaskQue
@JsonProperty("chatHandlerTimeout") @Nullable Duration chatHandlerTimeout,
@JsonProperty("chatHandlerNumRetries") @Nullable Integer chatHandlerNumRetries,
@JsonProperty("maxNumSegmentsToMerge") @Nullable Integer maxNumSegmentsToMerge,
@JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
@JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
super(
@ -75,7 +76,8 @@ public class UserCompactionTaskQueryTuningConfig extends ClientCompactionTaskQue
chatHandlerTimeout,
chatHandlerNumRetries,
maxNumSegmentsToMerge,
totalNumMergeTasks
totalNumMergeTasks,
maxColumnsToMerge
);
}

8
server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
View File

@ -141,6 +141,7 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
null,
null,
null,
null,
null
),
null,
@ -189,6 +190,7 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
null,
null,
null,
null,
null
),
null,
@ -242,7 +244,8 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
new Duration(3000L),
7,
1000,
100
100,
2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);
@ -282,7 +285,8 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
new Duration(3000L),
7,
1000,
100
100,
2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);

4
server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java
View File

@ -62,6 +62,7 @@ public class UserCompactionTaskQueryTuningConfigTest
null,
null,
null,
null,
null
);
final String json = OBJECT_MAPPER.writeValueAsString(config);
@ -103,7 +104,8 @@ public class UserCompactionTaskQueryTuningConfigTest
new Duration(3000L),
7,
1000,
100
100,
2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);

16
server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
View File

@ -736,6 +736,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -801,6 +802,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -860,6 +862,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -919,6 +922,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@ -986,6 +990,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1048,6 +1053,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1109,6 +1115,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, true),
@ -1211,6 +1218,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@ -1337,6 +1345,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1399,6 +1408,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1465,6 +1475,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1612,6 +1623,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1709,6 +1721,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@ -1774,6 +1787,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -1836,6 +1850,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,
@ -2154,6 +2169,7 @@ public class CompactSegmentsTest
null,
null,
null,
null,
null
),
null,

8
server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java
View File

@ -133,6 +133,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -177,6 +178,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -221,6 +223,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -265,6 +268,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -309,6 +313,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -353,6 +358,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -397,6 +403,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,
@ -441,6 +448,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
null,
null
),
null,

2
server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java
View File

@ -1538,6 +1538,7 @@ public class NewestSegmentFirstPolicyTest
null,
null,
null,
null,
null
),
null
@ -1572,6 +1573,7 @@ public class NewestSegmentFirstPolicyTest
null,
null,
null,
null,
null
),
null