Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update task memory computation in start-druid (#13563) 

Changes:
* Use 80% of memory specified for running services (versus 50% earlier).
* Tasks get either 512m / 1024m or 2048m now (versus 512m or 2048m earlier). 
* Add direct memory for router.
This commit is contained in:
Rishabh Singh 2022-12-15 11:06:16 +05:30 committed by GitHub
parent 2b605aa9cf
commit 97bc0220c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
examples/bin/start-druid-main.py
View File

@ -37,9 +37,11 @@ TASK_JAVA_OPTS_PROPERTY = "druid.indexer.runner.javaOptsArray"
TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity"
TASK_COUNT = "task-count" TASK_COUNT = "task-count"
TASK_MEM_TYPE_LOW = "low" TASK_MEM_TYPE_LOW = "low"
TASK_MEM_TYPE_MEDIUM = "medium"
TASK_MEM_TYPE_HIGH = "high" TASK_MEM_TYPE_HIGH = "high"
TASK_MEM_MAP = { TASK_MEM_MAP = {
TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"], TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"],
TASK_MEM_TYPE_MEDIUM: ["-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=512m"],
TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"]
} }
@ -82,7 +84,7 @@ SERVICE_MEMORY_RATIO = {
MINIMUM_MEMORY_MB = { MINIMUM_MEMORY_MB = {
MIDDLE_MANAGER: 64, MIDDLE_MANAGER: 64,
ROUTER: 128, ROUTER: 256,
TASKS: 1024, TASKS: 1024,
BROKER: 900, BROKER: 900,
COORDINATOR: 256, COORDINATOR: 256,
@ -431,10 +433,7 @@ def check_memory_constraint(total_memory, services):
if total_memory < required_memory: if total_memory < required_memory:
raise ValueError('Minimum memory required for starting services is {0}m'.format(required_memory)) raise ValueError('Minimum memory required for starting services is {0}m'.format(required_memory))
if total_memory >= 2 * lower_bound_memory: return int(total_memory * 0.8)
return int(total_memory / 2)
else:
return lower_bound_memory
def build_mm_task_java_opts_array(memory_type): def build_mm_task_java_opts_array(memory_type):
@ -457,6 +456,10 @@ def compute_tasks_memory(allocated_memory):
task_count = int(allocated_memory / 2048) task_count = int(allocated_memory / 2048)
memory_type = TASK_MEM_TYPE_HIGH memory_type = TASK_MEM_TYPE_HIGH
task_memory_mb = 2048 task_memory_mb = 2048
elif allocated_memory >= 2048:
task_count = int(allocated_memory / 1024)
memory_type = TASK_MEM_TYPE_MEDIUM
task_memory_mb = 1024
else: else:
task_count = 2 task_count = 2
memory_type = TASK_MEM_TYPE_LOW memory_type = TASK_MEM_TYPE_LOW
@ -483,6 +486,8 @@ def build_memory_config(service, allocated_memory):
else: else:
heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory
direct_memory = int(allocated_memory - heap_memory) direct_memory = int(allocated_memory - heap_memory)
if service == ROUTER:
direct_memory = 128
heap_memory = int(heap_memory) heap_memory = int(heap_memory)
if direct_memory == 0: if direct_memory == 0: