From a5c9c2950fdfb1012eb1e758eda31e15f040e6ae Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 13 Aug 2019 12:10:26 -0700 Subject: [PATCH] Add missing maxBytesInMemory in tuningConfig for auto compaction (#8274) * Add missing tuningConfigs for auto compaciton * Add doc * add test --- docs/content/configuration/index.md | 1 + .../ClientCompactQueryTuningConfig.java | 36 +++++++++++++--- .../DataSourceCompactionConfig.java | 13 +++++- .../DataSourceCompactionConfigTest.java | 41 ++++++++++++++++--- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index fcccac7c54b..134ff18918d 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -876,6 +876,7 @@ If you see this problem, it's recommended to set `skipOffsetFromLatest` to some |Property|Description|Required| |--------|-----------|--------| |`maxRowsInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 1000000)| +|`maxBytesInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (1/6 of max JVM memory)| |`maxTotalRows`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 20000000)| |`indexSpec`|See [IndexSpec](../ingestion/native_tasks.html#indexspec)|no| |`maxPendingPersists`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 0 (meaning one persist can be running concurrently with ingestion, and none can be queued up))| diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java index 14cf45b55a8..343bf49e445 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java @@ -32,9 +32,11 @@ public class ClientCompactQueryTuningConfig @Nullable private final Integer maxRowsPerSegment; @Nullable + private final Long maxBytesInMemory; + @Nullable private final Integer maxRowsInMemory; @Nullable - private final Integer maxTotalRows; + private final Long maxTotalRows; @Nullable private final IndexSpec indexSpec; @Nullable @@ -50,9 +52,12 @@ public class ClientCompactQueryTuningConfig return new ClientCompactQueryTuningConfig( maxRowsPerSegment, userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxRowsInMemory(), + userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxBytesInMemory(), userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxTotalRows(), userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getIndexSpec(), - userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(), + userCompactionTaskQueryTuningConfig == null + ? null + : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(), userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getPushTimeout() ); } @@ -61,13 +66,15 @@ public class ClientCompactQueryTuningConfig public ClientCompactQueryTuningConfig( @JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment, @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory, - @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows, + @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory, + @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows, @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec, @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists, @JsonProperty("pushTimeout") @Nullable Long pushTimeout ) { this.maxRowsPerSegment = maxRowsPerSegment; + this.maxBytesInMemory = maxBytesInMemory; this.maxRowsInMemory = maxRowsInMemory; this.maxTotalRows = maxTotalRows; this.indexSpec = indexSpec; @@ -97,7 +104,14 @@ public class ClientCompactQueryTuningConfig @JsonProperty @Nullable - public Integer getMaxTotalRows() + public Long getMaxBytesInMemory() + { + return maxBytesInMemory; + } + + @JsonProperty + @Nullable + public Long getMaxTotalRows() { return maxTotalRows; } @@ -134,6 +148,7 @@ public class ClientCompactQueryTuningConfig } ClientCompactQueryTuningConfig that = (ClientCompactQueryTuningConfig) o; return Objects.equals(maxRowsPerSegment, that.maxRowsPerSegment) && + Objects.equals(maxBytesInMemory, that.maxBytesInMemory) && Objects.equals(maxRowsInMemory, that.maxRowsInMemory) && Objects.equals(maxTotalRows, that.maxTotalRows) && Objects.equals(indexSpec, that.indexSpec) && @@ -144,14 +159,23 @@ public class ClientCompactQueryTuningConfig @Override public int hashCode() { - return Objects.hash(maxRowsPerSegment, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout); + return Objects.hash( + maxRowsPerSegment, + maxBytesInMemory, + maxRowsInMemory, + maxTotalRows, + indexSpec, + maxPendingPersists, + pushTimeout + ); } @Override public String toString() { - return getClass().getSimpleName() + "{" + + return "ClientCompactQueryTuningConfig{" + "maxRowsPerSegment=" + maxRowsPerSegment + + ", maxBytesInMemory=" + maxBytesInMemory + ", maxRowsInMemory=" + maxRowsInMemory + ", maxTotalRows=" + maxTotalRows + ", indexSpec=" + indexSpec + diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java index 09f4e81136d..ebf4da63da2 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java @@ -237,13 +237,22 @@ public class DataSourceCompactionConfig @JsonCreator public UserCompactTuningConfig( @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory, - @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows, + @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory, + @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows, @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec, @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists, @JsonProperty("pushTimeout") @Nullable Long pushTimeout ) { - super(null, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout); + super( + null, + maxRowsInMemory, + maxBytesInMemory, + maxTotalRows, + indexSpec, + maxPendingPersists, + pushTimeout + ); } @Override diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java index 9b8dbbda7e6..1f7676940d9 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java @@ -22,6 +22,10 @@ package org.apache.druid.server.coordinator; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import org.apache.druid.segment.data.CompressionStrategy; +import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.server.coordinator.DataSourceCompactionConfig.UserCompactTuningConfig; import org.joda.time.Period; import org.junit.Assert; @@ -97,7 +101,7 @@ public class DataSourceCompactionConfigTest @Test public void testSerdeUserCompactTuningConfig() throws IOException { - final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null); + final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null, null); final String json = objectMapper.writeValueAsString(config); // Check maxRowsPerSegment doesn't exist in the JSON string Assert.assertFalse(json.contains("maxRowsPerSegment")); @@ -118,7 +122,8 @@ public class DataSourceCompactionConfigTest new Period(3600), new UserCompactTuningConfig( null, - 10000, + null, + 10000L, null, null, null @@ -140,7 +145,7 @@ public class DataSourceCompactionConfigTest } @Test - public void testTargetCompactionSizeBytesWithMaxRowsPerSegment() + public void testSerdeTargetCompactionSizeBytesWithMaxRowsPerSegment() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( @@ -160,7 +165,7 @@ public class DataSourceCompactionConfigTest } @Test - public void testTargetCompactionSizeBytesWithMaxTotalRows() + public void testSerdeTargetCompactionSizeBytesWithMaxTotalRows() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( @@ -176,7 +181,8 @@ public class DataSourceCompactionConfigTest new Period(3600), new UserCompactTuningConfig( null, - 10000, + null, + 10000L, null, null, null @@ -198,7 +204,8 @@ public class DataSourceCompactionConfigTest new Period(3600), new UserCompactTuningConfig( null, - 10000, + null, + 10000L, null, null, null @@ -219,4 +226,26 @@ public class DataSourceCompactionConfigTest Assert.assertEquals(config.getTuningConfig(), fromJson.getTuningConfig()); Assert.assertEquals(config.getTaskContext(), fromJson.getTaskContext()); } + + @Test + public void testSerdeUserCompactionTuningConfig() throws IOException + { + final UserCompactTuningConfig tuningConfig = new UserCompactTuningConfig( + 1000, + 10000L, + 2000L, + new IndexSpec( + new RoaringBitmapSerdeFactory(false), + CompressionStrategy.LZF, + CompressionStrategy.UNCOMPRESSED, + LongEncodingStrategy.LONGS + ), + 1, + 3000L + ); + + final String json = objectMapper.writeValueAsString(tuningConfig); + final UserCompactTuningConfig fromJson = objectMapper.readValue(json, UserCompactTuningConfig.class); + Assert.assertEquals(tuningConfig, fromJson); + } }