From 10aab7506e13cb60bddd9f3c29c04079e2833cb2 Mon Sep 17 00:00:00 2001
From: Katya Macedo <38017980+ektravel@users.noreply.github.com>
Date: Wed, 11 Oct 2023 16:45:05 -0500
Subject: [PATCH] Dynamic configuration API documentation refactor (#15098)
Co-authored-by: demo-kratia <56242907+demo-kratia@users.noreply.github.com>
---
.../dynamic-configuration-api.md | 648 +++++++++++++++++-
1 file changed, 619 insertions(+), 29 deletions(-)
diff --git a/docs/api-reference/dynamic-configuration-api.md b/docs/api-reference/dynamic-configuration-api.md
index 762895abb6c..00d49cd5a31 100644
--- a/docs/api-reference/dynamic-configuration-api.md
+++ b/docs/api-reference/dynamic-configuration-api.md
@@ -3,73 +3,663 @@ id: dynamic-configuration-api
title: Dynamic configuration API
sidebar_label: Dynamic configuration
---
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
-This document describes the API endpoints to retrieve and manage the dynamic configurations for the [Coordinator](../configuration/index.md#overlord-dynamic-configuration) and [Overlord](../configuration/index.md#dynamic-configuration) in Apache Druid.
+This document describes the API endpoints to retrieve and manage dynamic configurations for the [Coordinator](../design/coordinator.md) and [Overlord](../design/overlord.md) in Apache Druid.
+
+In this topic, `http://ROUTER_IP:ROUTER_PORT` is a placeholder for your Router service address and port.
+Replace it with the information for your deployment.
+For example, use `http://localhost:8888` for quickstart deployments.
## Coordinator dynamic configuration
-See [Coordinator Dynamic Configuration](../configuration/index.md#dynamic-configuration) for details.
+The Coordinator has dynamic configurations to tune certain behavior on the fly, without requiring a service restart.
+For information on the supported properties, see [Coordinator dynamic configuration](../configuration/index.md#dynamic-configuration).
-Note that all _interval_ URL parameters are ISO 8601 strings delimited by a `_` instead of a `/`
-as in `2016-06-27_2016-06-28`.
+### Get dynamic configuration
-`GET /druid/coordinator/v1/config`
+Retrieves the current Coordinator dynamic configuration. Returns a JSON object with the dynamic configuration properties.
-Retrieves current coordinator dynamic configuration.
+#### URL
-`GET /druid/coordinator/v1/config/history?interval={interval}&count={count}`
+GET /druid/coordinator/v1/config
-Retrieves history of changes to overlord dynamic configuration. Accepts `interval` and `count` query string parameters
-to filter by interval and limit the number of results respectively.
+#### Responses
-`POST /druid/coordinator/v1/config`
+
-Update overlord dynamic worker configuration.
+
+*Successfully retrieved dynamic configuration*
+
+
+
+
+---
+
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config"
+```
+
+
+
+
+
+```HTTP
+GET /druid/coordinator/v1/config HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+
+Click to show sample response
+
+```json
+{
+ "millisToWaitBeforeDeleting": 900000,
+ "mergeBytesLimit": 524288000,
+ "mergeSegmentsLimit": 100,
+ "maxSegmentsToMove": 100,
+ "replicantLifetime": 15,
+ "replicationThrottleLimit": 500,
+ "balancerComputeThreads": 1,
+ "killDataSourceWhitelist": [],
+ "killPendingSegmentsSkipList": [],
+ "maxSegmentsInNodeLoadingQueue": 500,
+ "decommissioningNodes": [],
+ "decommissioningMaxPercentOfMaxSegmentsToMove": 70,
+ "pauseCoordination": false,
+ "replicateAfterLoadTimeout": false,
+ "maxNonPrimaryReplicantsToLoad": 2147483647,
+ "useRoundRobinSegmentAssignment": true,
+ "smartSegmentLoading": true,
+ "debugDimensions": null
+}
+```
+
+
+
+### Update dynamic configuration
+
+Submits a JSON-based dynamic configuration spec to the Coordinator.
+For information on the supported properties, see [Dynamic configuration](../configuration/index.md#dynamic-configuration).
+
+#### URL
+
+POST /druid/coordinator/v1/config
+
+#### Header parameters
+
+The endpoint supports a set of optional header parameters to populate the `author` and `comment` fields in the configuration history.
+
+* `X-Druid-Author`
+ * Type: String
+ * Author of the configuration change.
+* `X-Druid-Comment`
+ * Type: String
+ * Description for the update.
+
+#### Responses
+
+
+
+
+
+
+*Successfully updated dynamic configuration*
+
+
+
+
+---
+
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config" \
+--header 'Content-Type: application/json' \
+--data '{
+ "millisToWaitBeforeDeleting": 900000,
+ "mergeBytesLimit": 524288000,
+ "mergeSegmentsLimit": 100,
+ "maxSegmentsToMove": 5,
+ "percentOfSegmentsToConsiderPerMove": 100,
+ "useBatchedSegmentSampler": true,
+ "replicantLifetime": 15,
+ "replicationThrottleLimit": 10,
+ "balancerComputeThreads": 1,
+ "emitBalancingStats": true,
+ "killDataSourceWhitelist": [],
+ "killPendingSegmentsSkipList": [],
+ "maxSegmentsInNodeLoadingQueue": 100,
+ "decommissioningNodes": [],
+ "decommissioningMaxPercentOfMaxSegmentsToMove": 70,
+ "pauseCoordination": false,
+ "replicateAfterLoadTimeout": false,
+ "maxNonPrimaryReplicantsToLoad": 2147483647,
+ "useRoundRobinSegmentAssignment": true
+}'
+```
+
+
+
+
+
+```HTTP
+POST /druid/coordinator/v1/config HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+Content-Type: application/json
+Content-Length: 683
+
+{
+ "millisToWaitBeforeDeleting": 900000,
+ "mergeBytesLimit": 524288000,
+ "mergeSegmentsLimit": 100,
+ "maxSegmentsToMove": 5,
+ "percentOfSegmentsToConsiderPerMove": 100,
+ "useBatchedSegmentSampler": true,
+ "replicantLifetime": 15,
+ "replicationThrottleLimit": 10,
+ "balancerComputeThreads": 1,
+ "emitBalancingStats": true,
+ "killDataSourceWhitelist": [],
+ "killPendingSegmentsSkipList": [],
+ "maxSegmentsInNodeLoadingQueue": 100,
+ "decommissioningNodes": [],
+ "decommissioningMaxPercentOfMaxSegmentsToMove": 70,
+ "pauseCoordination": false,
+ "replicateAfterLoadTimeout": false,
+ "maxNonPrimaryReplicantsToLoad": 2147483647,
+ "useRoundRobinSegmentAssignment": true
+}
+```
+
+
+
+
+#### Sample response
+
+A successful request returns an HTTP `200 OK` message code and an empty response body.
+
+### Get dynamic configuration history
+
+Retrieves the history of changes to Coordinator dynamic configuration over an interval of time. Returns an empty array if there are no history records available.
+
+#### URL
+
+GET /druid/coordinator/v1/config/history
+
+#### Query parameters
+
+The endpoint supports a set of optional query parameters to filter results.
+
+* `interval`
+ * Type: String
+ * Limit the results to the specified time interval in ISO 8601 format delimited with `/`. For example, `2023-07-13/2023-07-19`. The default interval is one week. You can change this period by setting `druid.audit.manager.auditHistoryMillis` in the `runtime.properties` file for the Coordinator.
+
+* `count`
+ * Type: Integer
+ * Limit the number of results to the last `n` entries.
+
+#### Responses
+
+
+
+
+
+
+*Successfully retrieved history*
+
+
+
+
+
+---
+
+#### Sample request
+
+The following example retrieves the dynamic configuration history between `2022-07-13` and `2024-07-19`. The response is limited to 10 entries.
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config/history?interval=2022-07-13%2F2024-07-19&count=10"
+```
+
+
+
+
+
+```HTTP
+GET /druid/coordinator/v1/config/history?interval=2022-07-13/2024-07-19&count=10 HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+
+ Click to show sample response
+
+```json
+[
+ {
+ "key": "coordinator.config",
+ "type": "coordinator.config",
+ "auditInfo": {
+ "author": "",
+ "comment": "",
+ "ip": "127.0.0.1"
+ },
+ "payload": "{\"millisToWaitBeforeDeleting\":900000,\"mergeBytesLimit\":524288000,\"mergeSegmentsLimit\":100,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}",
+ "auditTime": "2023-10-03T20:59:51.622Z"
+ }
+]
+```
+
+
## Overlord dynamic configuration
-See [Overlord Dynamic Configuration](../configuration/index.md#overlord-dynamic-configuration) for details.
+The Overlord has dynamic configurations to tune how Druid assigns tasks to workers.
+For information on the supported properties, see [Overlord dynamic configuration](../configuration/index.md#overlord-dynamic-configuration).
-Note that all _interval_ URL parameters are ISO 8601 strings delimited by a `_` instead of a `/`
-as in `2016-06-27_2016-06-28`.
+### Get dynamic configuration
-`GET /druid/indexer/v1/worker`
+Retrieves the current Overlord dynamic configuration.
+Returns a JSON object with the dynamic configuration properties.
+Returns an empty response body if there is no current Overlord dynamic configuration.
-Retrieves current overlord dynamic configuration.
+#### URL
-`GET /druid/indexer/v1/worker/history?interval={interval}&count={count}`
+GET /druid/indexer/v1/worker
-Retrieves history of changes to overlord dynamic configuration. Accepts `interval` and `count` query string parameters
-to filter by interval and limit the number of results respectively.
+#### Responses
-`GET /druid/indexer/v1/workers`
+
-Retrieves a list of all the worker nodes in the cluster along with its metadata.
+
-`GET /druid/indexer/v1/scaling`
-Retrieves overlord scaling events if auto-scaling runners are in use.
+*Successfully retrieved dynamic configuration*
-`POST /druid/indexer/v1/worker`
+
+
-Update overlord dynamic worker configuration.
\ No newline at end of file
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/worker"
+```
+
+
+
+
+
+```HTTP
+GET /druid/indexer/v1/worker HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+
+ Click to show sample response
+
+```json
+{
+ "type": "default",
+ "selectStrategy": {
+ "type": "fillCapacityWithCategorySpec",
+ "workerCategorySpec": {
+ "categoryMap": {},
+ "strong": true
+ }
+ },
+ "autoScaler": null
+}
+```
+
+
+
+### Update dynamic configuration
+
+Submits a JSON-based dynamic configuration spec to the Overlord.
+For information on the supported properties, see [Overlord dynamic configuration](../configuration/index.md#overlord-dynamic-configuration).
+
+#### URL
+
+POST/druid/indexer/v1/worker
+
+#### Header parameters
+
+The endpoint supports a set of optional header parameters to populate the `author` and `comment` fields in the configuration history.
+
+* `X-Druid-Author`
+ * Type: String
+ * Author of the configuration change.
+* `X-Druid-Comment`
+ * Type: String
+ * Description for the update.
+
+#### Responses
+
+
+
+
+
+
+*Successfully updated dynamic configuration*
+
+
+
+
+---
+
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/worker" \
+--header 'Content-Type: application/json' \
+--data '{
+ "type": "default",
+ "selectStrategy": {
+ "type": "fillCapacityWithCategorySpec",
+ "workerCategorySpec": {
+ "categoryMap": {},
+ "strong": true
+ }
+ },
+ "autoScaler": null
+}'
+```
+
+
+
+
+
+```HTTP
+POST /druid/indexer/v1/worker HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+Content-Type: application/json
+Content-Length: 196
+
+{
+ "type": "default",
+ "selectStrategy": {
+ "type": "fillCapacityWithCategorySpec",
+ "workerCategorySpec": {
+ "categoryMap": {},
+ "strong": true
+ }
+ },
+ "autoScaler": null
+}
+```
+
+
+
+
+#### Sample response
+
+A successful request returns an HTTP `200 OK` message code and an empty response body.
+
+### Get dynamic configuration history
+
+Retrieves the history of changes to Overlord dynamic configuration over an interval of time. Returns an empty array if there are no history records available.
+
+#### URL
+
+GET /druid/indexer/v1/worker/history
+
+
+#### Query parameters
+
+The endpoint supports a set of optional query parameters to filter results.
+
+* `interval`
+ * Type: String
+ * Limit the results to the specified time interval in ISO 8601 format delimited with `/`. For example, `2023-07-13/2023-07-19`. The default interval is one week. You can change this period by setting `druid.audit.manager.auditHistoryMillis` in the `runtime.properties` file for the Overlord.
+
+* `count`
+ * Type: Integer
+ * Limit the number of results to the last `n` entries.
+
+#### Responses
+
+
+
+
+
+
+*Successfully retrieved history*
+
+
+
+
+---
+
+#### Sample request
+
+The following example retrieves the dynamic configuration history between `2022-07-13` and `2024-07-19`. The response is limited to 10 entries.
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/worker/history?interval=2022-07-13%2F2024-07-19&count=10"
+```
+
+
+
+
+
+```HTTP
+GET /druid/indexer/v1/worker/history?interval=2022-07-13%2F2024-07-19&count=10 HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+
+ Click to show sample response
+
+```json
+[
+ {
+ "key": "worker.config",
+ "type": "worker.config",
+ "auditInfo": {
+ "author": "",
+ "comment": "",
+ "ip": "127.0.0.1"
+ },
+ "payload": "{\"type\":\"default\",\"selectStrategy\":{\"type\":\"fillCapacityWithCategorySpec\",\"workerCategorySpec\":{\"categoryMap\":{},\"strong\":true}},\"autoScaler\":null}",
+ "auditTime": "2023-10-03T21:49:49.991Z"
+ }
+]
+```
+
+
+
+### Get an array of worker nodes in the cluster
+
+Returns an array of all the worker nodes in the cluster along with its corresponding metadata.
+
+GET/druid/indexer/v1/workers
+
+#### Responses
+
+
+
+
+
+
+*Successfully retrieved worker nodes*
+
+
+
+
+---
+
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/workers"
+```
+
+
+
+
+
+```HTTP
+GET /druid/indexer/v1/workers HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+
+ Click to show sample response
+
+```json
+[
+ {
+ "worker": {
+ "scheme": "http",
+ "host": "localhost:8091",
+ "ip": "198.51.100.0",
+ "capacity": 2,
+ "version": "0",
+ "category": "_default_worker_category"
+ },
+ "currCapacityUsed": 0,
+ "currParallelIndexCapacityUsed": 0,
+ "availabilityGroups": [],
+ "runningTasks": [],
+ "lastCompletedTaskTime": "2023-09-29T19:13:05.505Z",
+ "blacklistedUntil": null
+ }
+]
+```
+
+
+
+### Get scaling events
+
+Returns Overlord scaling events if autoscaling runners are in use.
+Returns an empty response body if there are no Overlord scaling events.
+
+#### URL
+
+GET/druid/indexer/v1/scaling
+
+#### Responses
+
+
+
+
+
+
+*Successfully retrieved scaling events*
+
+
+
+
+---
+
+#### Sample request
+
+
+
+
+
+
+```shell
+curl "http://ROUTER_IP:ROUTER_PORT/druid/indexer/v1/scaling"
+```
+
+
+
+
+
+```HTTP
+GET /druid/indexer/v1/scaling HTTP/1.1
+Host: http://ROUTER_IP:ROUTER_PORT
+```
+
+
+
+
+#### Sample response
+
+A successful request returns a `200 OK` response and an array of scaling events.