Fix usage of maxColumnsToMerge in auto-compaction tuning config (#12551)

Issue: 
Even though `CompactionTuningConfig` allows a `maxColumnsToMerge` config
(to optimize memory usage, particulary for datasources with many dimensions),
the corresponding client object `ClientCompactionTaskQueryTuningConfig`
(used by the coordinator duty `CompactSegments` to trigger auto-compaction)
does not contain this field. Thus, the value of `maxColumnsToMerge` specified
in any datasource compaction config is ignored.

Changes:
- Add field `maxColumnsToMerge` in `ClientCompactionTaskQueryTuningConfig`
  and `UserCompactionTaskQueryTuningConfig`
- Fix tests

indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java

@@ -122,7 +122,8 @@ public class ClientCompactionTaskQuerySerdeTest
             new Duration(3000L),
             7,
             1000,
-            100
+            100,
+            2
         ),
         new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true),
         new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))),
@@ -287,7 +288,7 @@ public class ClientCompactionTaskQuerySerdeTest
                 null,
                 null,
                 null,
-                null,
+                2,
                 null,
                 null
             )
@@ -342,7 +343,8 @@ public class ClientCompactionTaskQuerySerdeTest
             new Duration(3000L),
             7,
             1000,
-            100
+            100,
+            2
         ),
         new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true),
         new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))),

integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java

@@ -67,7 +67,8 @@ public class CompactionUtil
             null,
             null,
             null,
-            1
+            1,
+            null
         ),
         null,
         null,

integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java

@@ -1632,7 +1632,8 @@ public class ITAutoCompactionTest extends AbstractIndexerTest
             null,
             null,
             null,
-            1
+            1,
+            null
         ),
         granularitySpec,
         dimensionsSpec,

integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java

@@ -95,7 +95,8 @@ public class ITAutoCompactionUpgradeTest extends AbstractIndexerTest
             null,
             null,
             null,
-            1
+            1,
+            null
         ),
         new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
         null,

server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java

@@ -75,6 +75,8 @@ public class ClientCompactionTaskQueryTuningConfig
   @Nullable
   private final Integer totalNumMergeTasks;
   @Nullable
+  private final Integer maxColumnsToMerge;
+  @Nullable
   private final AppendableIndexSpec appendableIndexSpec;
 
   public static ClientCompactionTaskQueryTuningConfig from(
@@ -103,6 +105,7 @@ public class ClientCompactionTaskQueryTuningConfig
           null,
           null,
           null,
+          null,
           null
       );
     } else {
@@ -128,7 +131,8 @@ public class ClientCompactionTaskQueryTuningConfig
           userCompactionTaskQueryTuningConfig.getChatHandlerTimeout(),
           userCompactionTaskQueryTuningConfig.getChatHandlerNumRetries(),
           userCompactionTaskQueryTuningConfig.getMaxNumSegmentsToMerge(),
-          userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks()
+          userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks(),
+          userCompactionTaskQueryTuningConfig.getMaxColumnsToMerge()
       );
     }
   }
@@ -153,7 +157,8 @@ public class ClientCompactionTaskQueryTuningConfig
       @JsonProperty("chatHandlerTimeout") @Nullable Duration chatHandlerTimeout,
       @JsonProperty("chatHandlerNumRetries") @Nullable Integer chatHandlerNumRetries,
       @JsonProperty("maxNumSegmentsToMerge") @Nullable Integer maxNumSegmentsToMerge,
-      @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
+      @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
+      @JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
   )
   {
     this.maxRowsPerSegment = maxRowsPerSegment;
@@ -175,6 +180,7 @@ public class ClientCompactionTaskQueryTuningConfig
     this.chatHandlerNumRetries = chatHandlerNumRetries;
     this.maxNumSegmentsToMerge = maxNumSegmentsToMerge;
     this.totalNumMergeTasks = totalNumMergeTasks;
+    this.maxColumnsToMerge = maxColumnsToMerge;
   }
 
   @JsonProperty
@@ -318,6 +324,13 @@ public class ClientCompactionTaskQueryTuningConfig
     return totalNumMergeTasks;
   }
 
+  @JsonProperty
+  @Nullable
+  public Integer getMaxColumnsToMerge()
+  {
+    return maxColumnsToMerge;
+  }
+
   @JsonProperty
   @Nullable
   public AppendableIndexSpec getAppendableIndexSpec()
@@ -353,6 +366,7 @@ public class ClientCompactionTaskQueryTuningConfig
            Objects.equals(chatHandlerNumRetries, that.chatHandlerNumRetries) &&
            Objects.equals(maxNumSegmentsToMerge, that.maxNumSegmentsToMerge) &&
            Objects.equals(totalNumMergeTasks, that.totalNumMergeTasks) &&
+           Objects.equals(maxColumnsToMerge, that.maxColumnsToMerge) &&
            Objects.equals(appendableIndexSpec, that.appendableIndexSpec);
   }
 
@@ -378,6 +392,7 @@ public class ClientCompactionTaskQueryTuningConfig
         chatHandlerNumRetries,
         maxNumSegmentsToMerge,
         totalNumMergeTasks,
+        maxColumnsToMerge,
         appendableIndexSpec
     );
   }
@@ -404,6 +419,7 @@ public class ClientCompactionTaskQueryTuningConfig
            ", chatHandlerNumRetries=" + chatHandlerNumRetries +
            ", maxNumSegmentsToMerge=" + maxNumSegmentsToMerge +
            ", totalNumMergeTasks=" + totalNumMergeTasks +
+           ", maxColumnsToMerge=" + maxColumnsToMerge +
            ", appendableIndexSpec=" + appendableIndexSpec +
            '}';
   }

server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java

@@ -53,7 +53,8 @@ public class UserCompactionTaskQueryTuningConfig extends ClientCompactionTaskQue
       @JsonProperty("chatHandlerTimeout") @Nullable Duration chatHandlerTimeout,
       @JsonProperty("chatHandlerNumRetries") @Nullable Integer chatHandlerNumRetries,
       @JsonProperty("maxNumSegmentsToMerge") @Nullable Integer maxNumSegmentsToMerge,
-      @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
+      @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
+      @JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
   )
   {
     super(
@@ -75,7 +76,8 @@ public class UserCompactionTaskQueryTuningConfig extends ClientCompactionTaskQue
         chatHandlerTimeout,
         chatHandlerNumRetries,
         maxNumSegmentsToMerge,
-        totalNumMergeTasks
+        totalNumMergeTasks,
+        maxColumnsToMerge
     );
   }
 

server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java

@@ -141,6 +141,7 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -189,6 +190,7 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -242,7 +244,8 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
         new Duration(3000L),
         7,
         1000,
-        100
+        100,
+        2
     );
 
     final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);
@@ -282,7 +285,8 @@ public class DataSourceCompactionConfigTest extends InitializedNullHandlingTest
         new Duration(3000L),
         7,
         1000,
-        100
+        100,
+        2
     );
 
     final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);

server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java

@@ -62,6 +62,7 @@ public class UserCompactionTaskQueryTuningConfigTest
             null,
             null,
             null,
+            null,
             null
         );
     final String json = OBJECT_MAPPER.writeValueAsString(config);
@@ -103,7 +104,8 @@ public class UserCompactionTaskQueryTuningConfigTest
         new Duration(3000L),
         7,
         1000,
-        100
+        100,
+        2
     );
 
     final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);

server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java

@@ -736,6 +736,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -801,6 +802,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -860,6 +862,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -919,6 +922,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@@ -986,6 +990,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1048,6 +1053,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1109,6 +1115,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, true),
@@ -1211,6 +1218,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@@ -1337,6 +1345,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1399,6 +1408,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1465,6 +1475,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1612,6 +1623,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1709,6 +1721,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null),
@@ -1774,6 +1787,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -1836,6 +1850,7 @@ public class CompactSegmentsTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null,
@@ -2154,6 +2169,7 @@ public class CompactSegmentsTest
                   null,
                   null,
                   null,
+                  null,
                   null
               ),
               null,

server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java

@@ -133,6 +133,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -177,6 +178,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -221,6 +223,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -265,6 +268,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -309,6 +313,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -353,6 +358,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -397,6 +403,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,
@@ -441,6 +448,7 @@ public class NewestSegmentFirstIteratorTest
             null,
             null,
             null,
+            null,
             null
         ),
         null,

server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java

@@ -1538,6 +1538,7 @@ public class NewestSegmentFirstPolicyTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null
@@ -1572,6 +1573,7 @@ public class NewestSegmentFirstPolicyTest
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             null