Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-17 07:25:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix indexTask to respect forceExtendableShardSpecs (#5509)

Browse Source 
* Fix indexTask to respect forceExtendableShardSpecs

* add comments
This commit is contained in:
Jihoon Son 2018-04-05 23:54:59 -07:00 committed by Gian Merlino
parent 723857699c
commit 298ed1755d
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java
View File

@ -524,16 +524,15 @@ public class IndexTask extends AbstractTask implements ChatHandler
private static boolean isGuaranteedRollup(IndexIOConfig ioConfig, IndexTuningConfig tuningConfig)
{
Preconditions.checkState(
!(tuningConfig.isForceGuaranteedRollup() &&
(tuningConfig.isForceExtendableShardSpecs() || ioConfig.isAppendToExisting())),
"Perfect rollup cannot be guaranteed with extendable shardSpecs"
!tuningConfig.isForceGuaranteedRollup() || !ioConfig.isAppendToExisting(),
"Perfect rollup cannot be guaranteed when appending to existing dataSources"
);
return tuningConfig.isForceGuaranteedRollup();
}
private static boolean isExtendableShardSpecs(IndexIOConfig ioConfig, IndexTuningConfig tuningConfig)
{
return !isGuaranteedRollup(ioConfig, tuningConfig);
return tuningConfig.isForceExtendableShardSpecs() || ioConfig.isAppendToExisting();
}
/**
@ -1249,7 +1248,19 @@ public class IndexTask extends AbstractTask implements ChatHandler
private final IndexSpec indexSpec;
private final File basePersistDirectory;
private final int maxPendingPersists;
/**
* This flag is to force to always use an extendableShardSpec (like {@link NumberedShardSpec} even if
* {@link #forceGuaranteedRollup} is set.
*/
private final boolean forceExtendableShardSpecs;
/**
* This flag is to force _perfect rollup mode_. {@link IndexTask} will scan the whole input data twice to 1) figure
* out proper shard specs for each segment and 2) generate segments. Note that perfect rollup mode basically assumes
* that no more data will be appended in the future. As a result, in perfect rollup mode, {@link NoneShardSpec} and
* {@link HashBasedNumberedShardSpec} are used for a single shard and two or shards, respectively.
*/
private final boolean forceGuaranteedRollup;
private final boolean reportParseExceptions;
private final long pushTimeout;
@ -1345,11 +1356,6 @@ public class IndexTask extends AbstractTask implements ChatHandler
this.pushTimeout = pushTimeout == null ? DEFAULT_PUSH_TIMEOUT : pushTimeout;
this.basePersistDirectory = basePersistDirectory;
Preconditions.checkArgument(
!(this.forceExtendableShardSpecs && this.forceGuaranteedRollup),
"Perfect rollup cannot be guaranteed with extendable shardSpecs"
);
this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory;
if (this.reportParseExceptions) {

2
indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java
View File

@ -222,7 +222,7 @@ public class IndexTaskTest
tmpDir,
null,
null,
createTuningConfig(2, null, true, false),
createTuningConfig(2, null, true, true),
false
),
null,