From b80f20f7694716e94a40ad8826e78d3fd0d659f9 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 16 Jul 2019 17:40:41 -0700 Subject: [PATCH] add missing dynamic coordinator configs (#8090) --- web-console/src/console-application.tsx | 12 -- .../coordinator-dynamic-config-dialog.tsx | 105 +++++++++++++++++- web-console/src/entry.ts | 17 ++- .../views/load-data-view/load-data-view.tsx | 1 + 4 files changed, 119 insertions(+), 16 deletions(-) diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx index 268fb92c081..636c706089b 100644 --- a/web-console/src/console-application.tsx +++ b/web-console/src/console-application.tsx @@ -25,7 +25,6 @@ import { HashRouter, Route, Switch } from 'react-router-dom'; import { ExternalLink, HeaderActiveTab, HeaderBar, Loader } from './components'; import { AppToaster } from './singletons/toaster'; -import { UrlBaser } from './singletons/url-baser'; import { QueryManager } from './utils'; import { DRUID_DOCS_API, DRUID_DOCS_SQL } from './variables'; import { @@ -45,9 +44,6 @@ type Capabilities = 'working-with-sql' | 'working-without-sql' | 'broken'; export interface ConsoleApplicationProps { hideLegacy: boolean; - baseURL?: string; - customHeaderName?: string; - customHeaderValue?: string; } export interface ConsoleApplicationState { @@ -125,14 +121,6 @@ export class ConsoleApplication extends React.PureComponent< capabilitiesLoading: true, }; - if (props.baseURL) { - axios.defaults.baseURL = props.baseURL; - UrlBaser.baseURL = props.baseURL; - } - if (props.customHeaderName && props.customHeaderValue) { - axios.defaults.headers.common[props.customHeaderName] = props.customHeaderValue; - } - this.capabilitiesQueryManager = new QueryManager({ processQuery: async () => { const capabilities = await ConsoleApplication.discoverCapabilities(); diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx index 233d5328802..7d7048768a1 100644 --- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Intent } from '@blueprintjs/core'; +import { Code, Intent } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import axios from 'axios'; import React from 'react'; @@ -138,50 +138,153 @@ export class CoordinatorDynamicConfigDialog extends React.PureComponent< { name: 'balancerComputeThreads', type: 'number', + defaultValue: 1, + info: ( + <> + Thread pool size for computing moving cost of segments in segment balancing. + Consider increasing this if you have a lot of segments and moving segments starts + to get stuck. + + ), }, { name: 'emitBalancingStats', type: 'boolean', + info: ( + <> + Boolean flag for whether or not we should emit balancing stats. This is an + expensive operation. + + ), }, { name: 'killAllDataSources', type: 'boolean', + info: ( + <> + Send kill tasks for ALL dataSources if property{' '} + druid.coordinator.kill.on is true. If this is set to true then{' '} + killDataSourceWhitelist must not be specified or be empty list. + + ), }, { name: 'killDataSourceWhitelist', type: 'string-array', + info: ( + <> + List of dataSources for which kill tasks are sent if property{' '} + druid.coordinator.kill.on is true. This can be a list of + comma-separated dataSources or a JSON array. + + ), }, { name: 'killPendingSegmentsSkipList', type: 'string-array', + info: ( + <> + List of dataSources for which pendingSegments are NOT cleaned up if property{' '} + druid.coordinator.kill.pendingSegments.on is true. This can be a list + of comma-separated dataSources or a JSON array. + + ), }, { name: 'maxSegmentsInNodeLoadingQueue', type: 'number', + defaultValue: 0, + info: ( + <> + The maximum number of segments that could be queued for loading to any given + server. This parameter could be used to speed up segments loading process, + especially if there are "slow" nodes in the cluster (with low loading speed) or if + too much segments scheduled to be replicated to some particular node (faster + loading could be preferred to better segments distribution). Desired value depends + on segments loading speed, acceptable replication time and number of nodes. Value + 1000 could be a start point for a rather big cluster. Default value is 0 (loading + queue is unbounded) + + ), }, { name: 'maxSegmentsToMove', type: 'number', + defaultValue: 5, + info: <>The maximum number of segments that can be moved at any given time., }, { name: 'mergeBytesLimit', type: 'size-bytes', + defaultValue: 524288000, + info: <>The maximum total uncompressed size in bytes of segments to merge., }, { name: 'mergeSegmentsLimit', type: 'number', + defaultValue: 100, + info: <>The maximum number of segments that can be in a single append task., }, { name: 'millisToWaitBeforeDeleting', type: 'number', + defaultValue: 900000, + info: ( + <> + How long does the Coordinator need to be active before it can start removing + (marking unused) segments in metadata storage. + + ), }, { name: 'replicantLifetime', type: 'number', + defaultValue: 15, + info: ( + <> + The maximum number of Coordinator runs for a segment to be replicated before we + start alerting. + + ), }, { name: 'replicationThrottleLimit', type: 'number', + defaultValue: 10, + info: <>The maximum number of segments that can be replicated at one time., + }, + { + name: 'decommissioningNodes', + type: 'string-array', + info: ( + <> + List of historical servers to 'decommission'. Coordinator will not assign new + segments to 'decommissioning' servers, and segments will be moved away from them + to be placed on non-decommissioning servers at the maximum rate specified by{' '} + decommissioningMaxPercentOfMaxSegmentsToMove. + + ), + }, + { + name: 'decommissioningMaxPercentOfMaxSegmentsToMove', + type: 'number', + defaultValue: 70, + info: ( + <> + The maximum number of segments that may be moved away from 'decommissioning' + servers to non-decommissioning (that is, active) servers during one Coordinator + run. This value is relative to the total maximum segment movements allowed during + one run which is determined by maxSegmentsToMove. If + decommissioningMaxPercentOfMaxSegmentsToMove is 0, segments will + neither be moved from or to 'decommissioning' servers, effectively putting them in + a sort of "maintenance" mode that will not participate in balancing or assignment + by load rules. Decommissioning can also become stalled if there are no available + active servers to place the segments. By leveraging the maximum percent of + decommissioning segment movements, an operator can prevent active servers from + overload by prioritizing balancing, or decrease decommissioning time instead. The + value should be between 0 and 100. + + ), }, ]} model={dynamicConfig} diff --git a/web-console/src/entry.ts b/web-console/src/entry.ts index c7043cd6f8b..fcd7163ef1f 100644 --- a/web-console/src/entry.ts +++ b/web-console/src/entry.ts @@ -16,6 +16,7 @@ * limitations under the License. */ +import axios from 'axios'; import 'brace'; // Import Ace editor and all the sub components used in the app import 'brace/ext/language_tools'; import 'brace/theme/solarized_dark'; @@ -28,6 +29,7 @@ import './ace-modes/dsql'; import './ace-modes/hjson'; import './bootstrap/react-table-defaults'; import { ConsoleApplication } from './console-application'; +import { UrlBaser } from './singletons/url-baser'; import './entry.scss'; @@ -40,6 +42,7 @@ interface ConsoleConfig { baseURL?: string; customHeaderName?: string; customHeaderValue?: string; + customHeaders?: Record; } const consoleConfig: ConsoleConfig = (window as any).consoleConfig; @@ -47,12 +50,20 @@ if (typeof consoleConfig.title === 'string') { window.document.title = consoleConfig.title; } +if (consoleConfig.baseURL) { + axios.defaults.baseURL = consoleConfig.baseURL; + UrlBaser.baseURL = consoleConfig.baseURL; +} +if (consoleConfig.customHeaderName && consoleConfig.customHeaderValue) { + axios.defaults.headers.common[consoleConfig.customHeaderName] = consoleConfig.customHeaderValue; +} +if (consoleConfig.customHeaders) { + Object.assign(axios.defaults.headers, consoleConfig.customHeaders); +} + ReactDOM.render( React.createElement(ConsoleApplication, { hideLegacy: Boolean(consoleConfig.hideLegacy), - baseURL: consoleConfig.baseURL, - customHeaderName: consoleConfig.customHeaderName, - customHeaderValue: consoleConfig.customHeaderValue, }) as any, container, ); diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 85200b7f61e..65c5fb78571 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -2669,6 +2669,7 @@ export class LoadDataView extends React.PureComponent { if (['index', 'index_parallel'].includes(deepGet(spec, 'type'))) {