Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add log statements for tmpStorageBytes in MSQ (#14449) 

* Add log statements for tmpStorageBytes in MSQ

* Add log

* Update log message
This commit is contained in:
Adarsh Sanjeev 2023-07-11 11:02:12 +05:30 committed by GitHub
parent 66cac08a52
commit 30a91be15a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/WorkerStorageParameters.java
View File

@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.inject.Injector;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.msq.indexing.error.MSQException;
import org.apache.druid.msq.indexing.error.NotEnoughTemporaryStorageFault;
@ -31,10 +32,10 @@ import java.util.Objects;
/**
* Class for determining the amount of temporary disk space to allocate to various purposes, given the per-worker limit.
* Similar to {@link WorkerMemoryParameters}, but for temporary disk space.
*
* <br>
* Currently only used to allocate disk space for intermediate output from super sorter storage, if intermediate super
* sorter storage is enabled.
*
* <br>
* If it is enabled, keeps {@link #MINIMUM_BASIC_OPERATIONS_BYTES} for miscellaneous operations and
* configures the super sorter to use {@link #SUPER_SORTER_TMP_STORAGE_USABLE_FRACTION} of the remaining space for
* intermediate files. If this value is less than {@link #MINIMUM_SUPER_SORTER_TMP_STORAGE_BYTES},
@ -42,6 +43,8 @@ import java.util.Objects;
*/
public class WorkerStorageParameters
{
private static final Logger log = new Logger(WorkerStorageParameters.class);
/**
* Fraction of temporary worker storage that can be allocated to super sorter intermediate files.
*/
@ -97,6 +100,8 @@ public class WorkerStorageParameters
Preconditions.checkArgument(tmpStorageBytesPerTask > 0, "Temporary storage bytes passed: [%s] should be > 0", tmpStorageBytesPerTask);
long intermediateSuperSorterStorageMaxLocalBytes = computeUsableStorage(tmpStorageBytesPerTask);
log.info("Intermediate super sorter local storage size: %d bytes", intermediateSuperSorterStorageMaxLocalBytes);
if (intermediateSuperSorterStorageMaxLocalBytes < MINIMUM_SUPER_SORTER_TMP_STORAGE_BYTES) {
throw new MSQException(
new NotEnoughTemporaryStorageFault(