Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support maxColumnsToMerge in supervisor tuningConfig (#17030) 

* support maxColumnsToMerge in supervisor specs

* remove log line

* fix style

* add docs

* fix unit tests
This commit is contained in:
George Shiqi Wu 2024-09-11 18:00:13 -04:00 committed by GitHub
parent 9e1544e9c4
commit 428f58cf15
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 127 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/ingestion/supervisor.md
View File

@ -214,6 +214,7 @@ For configuration properties specific to Kafka and Kinesis, see [Kafka tuning co
|`logParseExceptions`|Boolean|If `true`, Druid logs an error message when a parsing exception occurs, containing information about the row where the error occurred.|No|`false`|
|`maxParseExceptions`|Integer|The maximum number of parse exceptions that can occur before the task halts ingestion and fails. Setting `reportParseExceptions` overrides this limit.|No|unlimited|
|`maxSavedParseExceptions`|Integer|When a parse exception occurs, Druid keeps track of the most recent parse exceptions. `maxSavedParseExceptions` limits the number of saved exception instances. These saved exceptions are available after the task finishes in the [task completion report](../ingestion/tasks.md#task-reports). Setting `reportParseExceptions` overrides this limit.|No|0|
|`maxColumnsToMerge`|Integer|Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.|No|-1|
## Start a supervisor

18
extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfig.java
View File

@ -74,7 +74,9 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
@Nullable Integer numPersistThreads,
@Nullable Integer recordBufferSize,
@Nullable Integer recordBufferOfferTimeout,
@Nullable Integer maxRecordsPerPoll)
@Nullable Integer maxRecordsPerPoll,
@Nullable Integer maxColumnsToMerge
)
{
super(
appendableIndexSpec,
@ -97,7 +99,8 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
this.recordBufferSize = recordBufferSize;
@ -130,7 +133,8 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
@JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads,
@JsonProperty("maxRecordsPerPoll") @Nullable Integer maxRecordsPerPoll
@JsonProperty("maxRecordsPerPoll") @Nullable Integer maxRecordsPerPoll,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
this(
@ -156,7 +160,9 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
numPersistThreads,
recordBufferSize,
recordBufferOfferTimeout,
maxRecordsPerPoll);
maxRecordsPerPoll,
maxColumnsToMerge
);
}
@Nullable
@ -226,7 +232,8 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
getNumPersistThreads(),
getRecordBufferSizeConfigured(),
getRecordBufferOfferTimeout(),
getMaxRecordsPerPollConfigured()
getMaxRecordsPerPollConfigured(),
getMaxColumnsToMerge()
);
}
@ -253,6 +260,7 @@ public class RabbitStreamIndexTaskTuningConfig extends SeekableStreamIndexTaskTu
", maxSavedParseExceptions=" + getMaxSavedParseExceptions() +
", numPersistThreads=" + getNumPersistThreads() +
", maxRecordsPerPole=" + getMaxRecordsPerPollConfigured() +
", maxColumnsToMerge=" + getMaxColumnsToMerge() +
'}';
}

12
extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTuningConfig.java
View File

@ -68,6 +68,7 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
null,
null,
null,
null,
null
);
}
@ -99,7 +100,9 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
@JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads,
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("maxRecordsPerPoll") @Nullable Integer maxRecordsPerPoll)
@JsonProperty("maxRecordsPerPoll") @Nullable Integer maxRecordsPerPoll,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
super(
appendableIndexSpec,
@ -124,7 +127,8 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
numPersistThreads,
recordBufferSize,
recordBufferOfferTimeout,
maxRecordsPerPoll
maxRecordsPerPoll,
maxColumnsToMerge
);
this.workerThreads = workerThreads;
this.chatRetries = (chatRetries != null ? chatRetries : DEFAULT_CHAT_RETRIES);
@ -210,6 +214,7 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
", maxSavedParseExceptions=" + getMaxSavedParseExceptions() +
", numPersistThreads=" + getNumPersistThreads() +
", maxRecordsPerPoll=" + getMaxRecordsPerPollConfigured() +
", maxColumnsToMerge=" + getMaxColumnsToMerge() +
'}';
}
@ -239,7 +244,8 @@ public class RabbitStreamSupervisorTuningConfig extends RabbitStreamIndexTaskTun
getRecordBufferSizeConfigured(),
getRecordBufferOfferTimeout(),
getMaxRecordsPerPollConfigured(),
getNumPersistThreads()
getNumPersistThreads(),
getMaxColumnsToMerge()
);
}
}

3
extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskTuningConfigTest.java
View File

@ -180,7 +180,8 @@ public class RabbitStreamIndexTaskTuningConfigTest
"maxParseExceptions=0, " +
"maxSavedParseExceptions=0, " +
"numPersistThreads=1, " +
"maxRecordsPerPoll=null}";
"maxRecordsPerPoll=null, " +
"maxColumnsToMerge=-1}";
Assert.assertEquals(resStr, config.toString());

4
extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java
View File

@ -158,7 +158,9 @@ public class RabbitStreamSupervisorTest extends EasyMockSupport
null,
null,
null,
100);
100,
null
);
rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
serviceEmitter = new StubServiceEmitter("RabbitStreamSupervisorTest", "localhost");
EmittingLogger.registerEmitter(serviceEmitter);

18
extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfig.java
View File

@ -52,7 +52,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
@Nullable Boolean logParseExceptions,
@Nullable Integer maxParseExceptions,
@Nullable Integer maxSavedParseExceptions,
@Nullable Integer numPersistThreads
@Nullable Integer numPersistThreads,
@Nullable Integer maxColumnsToMerge
)
{
super(
@ -76,7 +77,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
}
@ -100,7 +102,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
@JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions,
@JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
this(
@ -123,7 +126,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
}
@ -150,7 +154,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
isLogParseExceptions(),
getMaxParseExceptions(),
getMaxSavedParseExceptions(),
getNumPersistThreads()
getNumPersistThreads(),
getMaxColumnsToMerge()
);
}
@ -177,7 +182,8 @@ public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningCon
", maxParseExceptions=" + getMaxParseExceptions() +
", maxSavedParseExceptions=" + getMaxSavedParseExceptions() +
", numPersistThreads=" + getNumPersistThreads() +
'}';
", getMaxColumnsToMerge=" + getMaxColumnsToMerge() +
'}';
}
}

10
extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java
View File

@ -65,6 +65,7 @@ public class KafkaSupervisorTuningConfig extends KafkaIndexTaskTuningConfig
null,
null,
null,
null,
null
);
}
@ -93,7 +94,8 @@ public class KafkaSupervisorTuningConfig extends KafkaIndexTaskTuningConfig
@JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions,
@JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
super(
@ -116,7 +118,8 @@ public class KafkaSupervisorTuningConfig extends KafkaIndexTaskTuningConfig
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
this.workerThreads = workerThreads;
this.chatRetries = (chatRetries != null ? chatRetries : DEFAULT_CHAT_RETRIES);
@ -229,7 +232,8 @@ public class KafkaSupervisorTuningConfig extends KafkaIndexTaskTuningConfig
isLogParseExceptions(),
getMaxParseExceptions(),
getMaxSavedParseExceptions(),
getNumPersistThreads()
getNumPersistThreads(),
getMaxColumnsToMerge()
);
}
}

1
extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
View File

@ -2861,6 +2861,7 @@ public class KafkaIndexTaskTest extends SeekableStreamIndexTaskTestBase
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
null,
null
);
if (!context.containsKey(SeekableStreamSupervisor.CHECKPOINTS_CTX_KEY)) {

12
extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java
View File

@ -73,6 +73,7 @@ public class KafkaIndexTaskTuningConfigTest
Assert.assertEquals(false, config.isReportParseExceptions());
Assert.assertEquals(Duration.ofMinutes(15).toMillis(), config.getHandoffConditionTimeout());
Assert.assertEquals(1, config.getNumPersistThreads());
Assert.assertEquals(-1, config.getMaxColumnsToMerge());
}
@Test
@ -123,6 +124,7 @@ public class KafkaIndexTaskTuningConfigTest
config.getIndexSpecForIntermediatePersists()
);
Assert.assertEquals(2, config.getNumPersistThreads());
Assert.assertEquals(-1, config.getMaxColumnsToMerge());
}
@Test
@ -152,7 +154,8 @@ public class KafkaIndexTaskTuningConfigTest
null,
null,
null,
2
2,
5
);
KafkaIndexTaskTuningConfig copy = original.convertToTaskTuningConfig();
@ -168,6 +171,7 @@ public class KafkaIndexTaskTuningConfigTest
Assert.assertEquals(true, copy.isReportParseExceptions());
Assert.assertEquals(5L, copy.getHandoffConditionTimeout());
Assert.assertEquals(2, copy.getNumPersistThreads());
Assert.assertEquals(5, copy.getMaxColumnsToMerge());
}
@Test
@ -193,7 +197,8 @@ public class KafkaIndexTaskTuningConfigTest
true,
42,
42,
2
2,
-1
);
String serialized = mapper.writeValueAsString(base);
@ -219,6 +224,7 @@ public class KafkaIndexTaskTuningConfigTest
Assert.assertEquals(base.getMaxParseExceptions(), deserialized.getMaxParseExceptions());
Assert.assertEquals(base.getMaxSavedParseExceptions(), deserialized.getMaxSavedParseExceptions());
Assert.assertEquals(base.getNumPersistThreads(), deserialized.getNumPersistThreads());
Assert.assertEquals(base.getMaxColumnsToMerge(), deserialized.getMaxColumnsToMerge());
}
@Test
@ -244,6 +250,7 @@ public class KafkaIndexTaskTuningConfigTest
42,
42,
2,
-1,
"extra string"
);
@ -269,6 +276,7 @@ public class KafkaIndexTaskTuningConfigTest
Assert.assertEquals(base.getMaxParseExceptions(), deserialized.getMaxParseExceptions());
Assert.assertEquals(base.getMaxSavedParseExceptions(), deserialized.getMaxSavedParseExceptions());
Assert.assertEquals(base.getNumPersistThreads(), deserialized.getNumPersistThreads());
Assert.assertEquals(base.getMaxColumnsToMerge(), deserialized.getMaxColumnsToMerge());
}
@Test

9
extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
View File

@ -346,7 +346,8 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null
null,
null
);
EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(kafkaSupervisorIOConfig).anyTimes();
@ -497,6 +498,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
),
null
@ -4221,6 +4223,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
)
);
@ -4260,6 +4263,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
);
@ -4413,6 +4417,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
)
);
@ -4888,6 +4893,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
10,
null,
null
);
@ -5002,6 +5008,7 @@ public class KafkaSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
);

4
extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/test/TestModifiedKafkaIndexTaskTuningConfig.java
View File

@ -56,6 +56,7 @@ public class TestModifiedKafkaIndexTaskTuningConfig extends KafkaIndexTaskTuning
@JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("numPersistThreads") @Nullable Integer numPersistThreads,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge,
@JsonProperty("extra") String extra
)
{
@ -79,7 +80,8 @@ public class TestModifiedKafkaIndexTaskTuningConfig extends KafkaIndexTaskTuning
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
this.extra = extra;
}

18
extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfig.java
View File

@ -92,7 +92,8 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
@Nullable Integer maxSavedParseExceptions,
@Deprecated @Nullable Integer maxRecordsPerPoll,
@Nullable Integer maxBytesPerPoll,
@Nullable Period intermediateHandoffPeriod
@Nullable Period intermediateHandoffPeriod,
@Nullable Integer maxColumnsToMerge
)
{
super(
@ -116,7 +117,8 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
null
null,
maxColumnsToMerge
);
this.recordBufferSize = recordBufferSize;
this.recordBufferSizeBytes = recordBufferSizeBytes;
@ -161,7 +163,8 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
@JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions,
@JsonProperty("maxRecordsPerPoll") @Deprecated @Nullable Integer maxRecordsPerPoll,
@JsonProperty("maxBytesPerPoll") @Nullable Integer maxBytesPerPoll,
@JsonProperty("intermediateHandoffPeriod") @Nullable Period intermediateHandoffPeriod
@JsonProperty("intermediateHandoffPeriod") @Nullable Period intermediateHandoffPeriod,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
this(
@ -191,7 +194,8 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
maxSavedParseExceptions,
maxRecordsPerPoll,
maxBytesPerPoll,
intermediateHandoffPeriod
intermediateHandoffPeriod,
maxColumnsToMerge
);
}
@ -294,7 +298,8 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
getMaxSavedParseExceptions(),
getMaxRecordsPerPollConfigured(),
getMaxBytesPerPollConfigured(),
getIntermediateHandoffPeriod()
getIntermediateHandoffPeriod(),
getMaxColumnsToMerge()
);
}
@ -363,6 +368,7 @@ public class KinesisIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningC
", maxRecordsPerPoll=" + maxRecordsPerPoll +
", maxBytesPerPoll=" + maxBytesPerPoll +
", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() +
'}';
", maxColumnsToMerge=" + getMaxColumnsToMerge() +
'}';
}
}

11
extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfig.java
View File

@ -76,6 +76,7 @@ public class KinesisSupervisorTuningConfig extends KinesisIndexTaskTuningConfig
null,
null,
null,
null,
null
);
}
@ -113,7 +114,8 @@ public class KinesisSupervisorTuningConfig extends KinesisIndexTaskTuningConfig
@JsonProperty("intermediateHandoffPeriod") Period intermediateHandoffPeriod,
@JsonProperty("repartitionTransitionDuration") Period repartitionTransitionDuration,
@JsonProperty("offsetFetchPeriod") Period offsetFetchPeriod,
@JsonProperty("useListShards") Boolean useListShards
@JsonProperty("useListShards") Boolean useListShards,
@JsonProperty("maxColumnsToMerge") Integer maxColumnsToMerge
)
{
super(
@ -143,7 +145,8 @@ public class KinesisSupervisorTuningConfig extends KinesisIndexTaskTuningConfig
maxSavedParseExceptions,
maxRecordsPerPoll,
maxBytesPerPoll,
intermediateHandoffPeriod
intermediateHandoffPeriod,
maxColumnsToMerge
);
this.workerThreads = workerThreads;
@ -244,6 +247,7 @@ public class KinesisSupervisorTuningConfig extends KinesisIndexTaskTuningConfig
", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() +
", repartitionTransitionDuration=" + getRepartitionTransitionDuration() +
", useListShards=" + isUseListShards() +
", maxColumnsToMerge=" + getMaxColumnsToMerge() +
'}';
}
@ -277,7 +281,8 @@ public class KinesisSupervisorTuningConfig extends KinesisIndexTaskTuningConfig
getMaxSavedParseExceptions(),
getMaxRecordsPerPollConfigured(),
getMaxBytesPerPollConfigured(),
getIntermediateHandoffPeriod()
getIntermediateHandoffPeriod(),
getMaxColumnsToMerge()
);
}
}

1
extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java
View File

@ -78,6 +78,7 @@ public class KinesisIndexTaskSerdeTest
null,
null,
null,
null,
null
);
private static final KinesisIndexTaskIOConfig IO_CONFIG = new KinesisIndexTaskIOConfig(

3
extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
View File

@ -2374,7 +2374,8 @@ public class KinesisIndexTaskTest extends SeekableStreamIndexTaskTestBase
maxSavedParseExceptions,
maxRecordsPerPoll,
maxBytesPerPoll,
intermediateHandoffPeriod
intermediateHandoffPeriod,
null
);
return createTask(taskId, dataSchema, ioConfig, tuningConfig, context);
}

12
extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java
View File

@ -134,6 +134,8 @@ public class KinesisIndexTaskTuningConfigTest
Assert.assertEquals(2, (int) config.getFetchThreads());
Assert.assertTrue(config.isSkipSequenceNumberAvailabilityCheck());
Assert.assertFalse(config.isResetOffsetAutomatically());
Assert.assertEquals(-1, config.getMaxColumnsToMerge());
}
@Test
@ -166,7 +168,8 @@ public class KinesisIndexTaskTuningConfigTest
500,
6000,
1_000_000,
new Period("P3D")
new Period("P3D"),
1000
);
String serialized = mapper.writeValueAsString(base);
@ -197,6 +200,7 @@ public class KinesisIndexTaskTuningConfigTest
Assert.assertEquals(base.getRecordBufferSizeBytesConfigured(), deserialized.getRecordBufferSizeBytesConfigured());
Assert.assertEquals(base.getMaxRecordsPerPollConfigured(), deserialized.getMaxRecordsPerPollConfigured());
Assert.assertEquals(base.getMaxBytesPerPollConfigured(), deserialized.getMaxBytesPerPollConfigured());
Assert.assertEquals(base.getMaxColumnsToMerge(), deserialized.getMaxColumnsToMerge());
}
@Test
@ -229,7 +233,8 @@ public class KinesisIndexTaskTuningConfigTest
500,
1_000_000,
6000,
new Period("P3D")
new Period("P3D"),
1000
);
String serialized = mapper.writeValueAsString(new TestModifiedKinesisIndexTaskTuningConfig(base, "loool"));
@ -257,6 +262,7 @@ public class KinesisIndexTaskTuningConfigTest
Assert.assertEquals(base.getRecordBufferOfferTimeout(), deserialized.getRecordBufferOfferTimeout());
Assert.assertEquals(base.getRecordBufferSizeBytesConfigured(), deserialized.getRecordBufferSizeBytesConfigured());
Assert.assertEquals(base.getMaxRecordsPerPollConfigured(), deserialized.getMaxRecordsPerPollConfigured());
Assert.assertEquals(base.getMaxColumnsToMerge(), deserialized.getMaxColumnsToMerge());
}
@Test
@ -322,6 +328,7 @@ public class KinesisIndexTaskTuningConfigTest
null,
null,
null,
null,
null
);
KinesisIndexTaskTuningConfig copy = original.convertToTaskTuningConfig();
@ -345,6 +352,7 @@ public class KinesisIndexTaskTuningConfigTest
Assert.assertTrue(copy.isResetOffsetAutomatically());
Assert.assertEquals(10, (int) copy.getMaxRecordsPerPollConfigured());
Assert.assertEquals(new Period().withDays(Integer.MAX_VALUE), copy.getIntermediateHandoffPeriod());
Assert.assertEquals(-1, copy.getMaxColumnsToMerge());
}
@Test

3
extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
View File

@ -207,6 +207,7 @@ public class KinesisSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
);
rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
@ -3980,6 +3981,7 @@ public class KinesisSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
);
@ -5159,6 +5161,7 @@ public class KinesisSupervisorTest extends EasyMockSupport
null,
null,
null,
null,
null
);

7
extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/test/TestModifiedKinesisIndexTaskTuningConfig.java
View File

@ -63,6 +63,7 @@ public class TestModifiedKinesisIndexTaskTuningConfig extends KinesisIndexTaskTu
@JsonProperty("maxRecordsPerPoll") @Nullable Integer maxRecordsPerPoll,
@JsonProperty("maxBytesPerPoll") @Nullable Integer maxBytesPerPoll,
@JsonProperty("intermediateHandoffPeriod") @Nullable Period intermediateHandoffPeriod,
@JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge,
@JsonProperty("extra") String extra
)
{
@ -93,7 +94,8 @@ public class TestModifiedKinesisIndexTaskTuningConfig extends KinesisIndexTaskTu
maxSavedParseExceptions,
maxRecordsPerPoll,
maxBytesPerPoll,
intermediateHandoffPeriod
intermediateHandoffPeriod,
maxColumnsToMerge
);
this.extra = extra;
}
@ -127,7 +129,8 @@ public class TestModifiedKinesisIndexTaskTuningConfig extends KinesisIndexTaskTu
base.getMaxSavedParseExceptions(),
base.getMaxRecordsPerPollConfigured(),
base.getMaxBytesPerPollConfigured(),
base.getIntermediateHandoffPeriod()
base.getIntermediateHandoffPeriod(),
base.getMaxColumnsToMerge()
);
this.extra = extra;
}

17
indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTuningConfig.java
View File

@ -41,6 +41,7 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
private static final IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.DEFAULT;
private static final Boolean DEFAULT_REPORT_PARSE_EXCEPTIONS = Boolean.FALSE;
private static final long DEFAULT_HANDOFF_CONDITION_TIMEOUT = Duration.ofMinutes(15).toMillis();
private static final int DEFAULT_MAX_COLUMNS_TO_MERGE = -1;
private final AppendableIndexSpec appendableIndexSpec;
private final int maxRowsInMemory;
@ -66,6 +67,7 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
private final int maxSavedParseExceptions;
private final int numPersistThreads;
private final int maxColumnsToMerge;
public SeekableStreamIndexTaskTuningConfig(
@Nullable AppendableIndexSpec appendableIndexSpec,
@ -88,7 +90,8 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
@Nullable Boolean logParseExceptions,
@Nullable Integer maxParseExceptions,
@Nullable Integer maxSavedParseExceptions,
@Nullable Integer numPersistThreads
@Nullable Integer numPersistThreads,
@Nullable Integer maxColumnsToMerge
)
{
this.appendableIndexSpec = appendableIndexSpec == null ? DEFAULT_APPENDABLE_INDEX : appendableIndexSpec;
@ -139,6 +142,7 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
: logParseExceptions;
this.numPersistThreads = numPersistThreads == null ?
DEFAULT_NUM_PERSIST_THREADS : Math.max(numPersistThreads, DEFAULT_NUM_PERSIST_THREADS);
this.maxColumnsToMerge = maxColumnsToMerge == null ? DEFAULT_MAX_COLUMNS_TO_MERGE : maxColumnsToMerge;
}
@Override
@ -289,6 +293,13 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
return numPersistThreads;
}
@Override
@JsonProperty
public int getMaxColumnsToMerge()
{
return maxColumnsToMerge;
}
@Override
public abstract SeekableStreamIndexTaskTuningConfig withBasePersistDirectory(File dir);
@ -315,6 +326,7 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
maxParseExceptions == that.maxParseExceptions &&
maxSavedParseExceptions == that.maxSavedParseExceptions &&
numPersistThreads == that.numPersistThreads &&
maxColumnsToMerge == that.maxColumnsToMerge &&
Objects.equals(partitionsSpec, that.partitionsSpec) &&
Objects.equals(intermediatePersistPeriod, that.intermediatePersistPeriod) &&
Objects.equals(basePersistDirectory, that.basePersistDirectory) &&
@ -347,7 +359,8 @@ public abstract class SeekableStreamIndexTaskTuningConfig implements Appenderato
logParseExceptions,
maxParseExceptions,
maxSavedParseExceptions,
numPersistThreads
numPersistThreads,
maxColumnsToMerge
);
}

1
indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java
View File

@ -522,6 +522,7 @@ public class SeekableStreamSupervisorSpecTest extends EasyMockSupport
null,
null,
null,
null,
null
)
{

1
indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
View File

@ -2683,6 +2683,7 @@ public class SeekableStreamSupervisorStateTest extends EasyMockSupport
null,
null,
null,
null,
null
)
{