Add missing maxBytesInMemory in tuningConfig for auto compaction (#8274)

* Add missing tuningConfigs for auto compaciton

* Add doc

* add test

docs/content/configuration/index.md

@@ -876,6 +876,7 @@ If you see this problem, it's recommended to set `skipOffsetFromLatest` to some
 |Property|Description|Required|
 |--------|-----------|--------|
 |`maxRowsInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 1000000)|
+|`maxBytesInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (1/6 of max JVM memory)|
 |`maxTotalRows`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 20000000)|
 |`indexSpec`|See [IndexSpec](../ingestion/native_tasks.html#indexspec)|no|
 |`maxPendingPersists`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 0 (meaning one persist can be running concurrently with ingestion, and none can be queued up))|

server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java

@@ -32,9 +32,11 @@ public class ClientCompactQueryTuningConfig
   @Nullable
   private final Integer maxRowsPerSegment;
   @Nullable
+  private final Long maxBytesInMemory;
+  @Nullable
   private final Integer maxRowsInMemory;
   @Nullable
-  private final Integer maxTotalRows;
+  private final Long maxTotalRows;
   @Nullable
   private final IndexSpec indexSpec;
   @Nullable
@@ -50,9 +52,12 @@ public class ClientCompactQueryTuningConfig
     return new ClientCompactQueryTuningConfig(
         maxRowsPerSegment,
         userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxRowsInMemory(),
+        userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxBytesInMemory(),
         userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxTotalRows(),
         userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getIndexSpec(),
-        userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(),
+        userCompactionTaskQueryTuningConfig == null
+        ? null
+        : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(),
         userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getPushTimeout()
     );
   }
@@ -61,13 +66,15 @@ public class ClientCompactQueryTuningConfig
   public ClientCompactQueryTuningConfig(
       @JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment,
       @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory,
-      @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows,
+      @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory,
+      @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows,
       @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
       @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
       @JsonProperty("pushTimeout") @Nullable Long pushTimeout
   )
   {
     this.maxRowsPerSegment = maxRowsPerSegment;
+    this.maxBytesInMemory = maxBytesInMemory;
     this.maxRowsInMemory = maxRowsInMemory;
     this.maxTotalRows = maxTotalRows;
     this.indexSpec = indexSpec;
@@ -97,7 +104,14 @@ public class ClientCompactQueryTuningConfig
 
   @JsonProperty
   @Nullable
-  public Integer getMaxTotalRows()
+  public Long getMaxBytesInMemory()
+  {
+    return maxBytesInMemory;
+  }
+
+  @JsonProperty
+  @Nullable
+  public Long getMaxTotalRows()
   {
     return maxTotalRows;
   }
@@ -134,6 +148,7 @@ public class ClientCompactQueryTuningConfig
     }
     ClientCompactQueryTuningConfig that = (ClientCompactQueryTuningConfig) o;
     return Objects.equals(maxRowsPerSegment, that.maxRowsPerSegment) &&
+           Objects.equals(maxBytesInMemory, that.maxBytesInMemory) &&
            Objects.equals(maxRowsInMemory, that.maxRowsInMemory) &&
            Objects.equals(maxTotalRows, that.maxTotalRows) &&
            Objects.equals(indexSpec, that.indexSpec) &&
@@ -144,14 +159,23 @@ public class ClientCompactQueryTuningConfig
   @Override
   public int hashCode()
   {
-    return Objects.hash(maxRowsPerSegment, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout);
+    return Objects.hash(
+        maxRowsPerSegment,
+        maxBytesInMemory,
+        maxRowsInMemory,
+        maxTotalRows,
+        indexSpec,
+        maxPendingPersists,
+        pushTimeout
+    );
   }
 
   @Override
   public String toString()
   {
-    return getClass().getSimpleName() + "{" +
+    return "ClientCompactQueryTuningConfig{" +
            "maxRowsPerSegment=" + maxRowsPerSegment +
+           ", maxBytesInMemory=" + maxBytesInMemory +
            ", maxRowsInMemory=" + maxRowsInMemory +
            ", maxTotalRows=" + maxTotalRows +
            ", indexSpec=" + indexSpec +

server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java

@@ -237,13 +237,22 @@ public class DataSourceCompactionConfig
     @JsonCreator
     public UserCompactTuningConfig(
         @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory,
-        @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows,
+        @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory,
+        @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows,
         @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
         @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
         @JsonProperty("pushTimeout") @Nullable Long pushTimeout
     )
     {
-      super(null, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout);
+      super(
+          null,
+          maxRowsInMemory,
+          maxBytesInMemory,
+          maxTotalRows,
+          indexSpec,
+          maxPendingPersists,
+          pushTimeout
+      );
     }
 
     @Override

server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java

@@ -22,6 +22,10 @@ package org.apache.druid.server.coordinator;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
 import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy;
+import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
 import org.apache.druid.server.coordinator.DataSourceCompactionConfig.UserCompactTuningConfig;
 import org.joda.time.Period;
 import org.junit.Assert;
@@ -97,7 +101,7 @@ public class DataSourceCompactionConfigTest
   @Test
   public void testSerdeUserCompactTuningConfig() throws IOException
   {
-    final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null);
+    final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null, null);
     final String json = objectMapper.writeValueAsString(config);
     // Check maxRowsPerSegment doesn't exist in the JSON string
     Assert.assertFalse(json.contains("maxRowsPerSegment"));
@@ -118,7 +122,8 @@ public class DataSourceCompactionConfigTest
         new Period(3600),
         new UserCompactTuningConfig(
             null,
-            10000,
+            null,
+            10000L,
             null,
             null,
             null
@@ -140,7 +145,7 @@ public class DataSourceCompactionConfigTest
   }
 
   @Test
-  public void testTargetCompactionSizeBytesWithMaxRowsPerSegment()
+  public void testSerdeTargetCompactionSizeBytesWithMaxRowsPerSegment()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage(
@@ -160,7 +165,7 @@ public class DataSourceCompactionConfigTest
   }
 
   @Test
-  public void testTargetCompactionSizeBytesWithMaxTotalRows()
+  public void testSerdeTargetCompactionSizeBytesWithMaxTotalRows()
   {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage(
@@ -176,7 +181,8 @@ public class DataSourceCompactionConfigTest
         new Period(3600),
         new UserCompactTuningConfig(
             null,
-            10000,
+            null,
+            10000L,
             null,
             null,
             null
@@ -198,7 +204,8 @@ public class DataSourceCompactionConfigTest
         new Period(3600),
         new UserCompactTuningConfig(
             null,
-            10000,
+            null,
+            10000L,
             null,
             null,
             null
@@ -219,4 +226,26 @@ public class DataSourceCompactionConfigTest
     Assert.assertEquals(config.getTuningConfig(), fromJson.getTuningConfig());
     Assert.assertEquals(config.getTaskContext(), fromJson.getTaskContext());
   }
+
+  @Test
+  public void testSerdeUserCompactionTuningConfig() throws IOException
+  {
+    final UserCompactTuningConfig tuningConfig = new UserCompactTuningConfig(
+        1000,
+        10000L,
+        2000L,
+        new IndexSpec(
+            new RoaringBitmapSerdeFactory(false),
+            CompressionStrategy.LZF,
+            CompressionStrategy.UNCOMPRESSED,
+            LongEncodingStrategy.LONGS
+        ),
+        1,
+        3000L
+    );
+
+    final String json = objectMapper.writeValueAsString(tuningConfig);
+    final UserCompactTuningConfig fromJson = objectMapper.readValue(json, UserCompactTuningConfig.class);
+    Assert.assertEquals(tuningConfig, fromJson);
+  }
 }