From d8f4353c43ccc2578b2c58488e10ff8dfb38b6c5 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 28 Nov 2022 16:50:38 -0800 Subject: [PATCH] Web console: be more robust to aux queries failing and improve kill tasks (#13431) * be more robust to aux queries failing * feedback fixes * remove empty block * fix spelling * remove killAllDataSources from the console --- .../src/components/auto-form/auto-form.scss | 12 - .../form-group-with-info.scss | 14 + .../table-clickable-cell.scss | 4 + .../table-clickable-cell.tsx | 11 +- .../warning-checklist/warning-checklist.tsx | 18 +- .../async-action-dialog.tsx | 2 +- web-console/src/dialogs/index.ts | 1 + .../kill-datasource-dialog.tsx | 110 +++++ .../compaction-status.spec.ts | 41 +- .../compaction-status/compaction-status.ts | 22 +- .../coordinator-dynamic-config.tsx | 13 +- .../datasources-view/datasources-view.tsx | 396 ++++++++++-------- .../src/views/services-view/services-view.tsx | 215 ++++++---- 13 files changed, 527 insertions(+), 332 deletions(-) create mode 100644 web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx diff --git a/web-console/src/components/auto-form/auto-form.scss b/web-console/src/components/auto-form/auto-form.scss index 36f8c34d92d..c303abc294f 100644 --- a/web-console/src/components/auto-form/auto-form.scss +++ b/web-console/src/components/auto-form/auto-form.scss @@ -16,19 +16,7 @@ * limitations under the License. */ -@import '../../variables'; - .auto-form { - // Popover in info label - label.#{$bp-ns}-label { - position: relative; - - .#{$bp-ns}-text-muted { - position: absolute; - right: 0; - } - } - .custom-input input { cursor: pointer; } diff --git a/web-console/src/components/form-group-with-info/form-group-with-info.scss b/web-console/src/components/form-group-with-info/form-group-with-info.scss index c9587cb088f..a64c6d29274 100644 --- a/web-console/src/components/form-group-with-info/form-group-with-info.scss +++ b/web-console/src/components/form-group-with-info/form-group-with-info.scss @@ -19,6 +19,20 @@ @import '../../variables'; .form-group-with-info { + label.#{$bp-ns}-label { + position: relative; + + .#{$bp-ns}-text-muted { + position: absolute; + right: 0; + + // This is only needed because BP4 alerts are too agro in setting CSS on icons + .#{$bp-ns}-icon { + margin-right: 0; + } + } + } + .#{$bp-ns}-text-muted .#{$bp-ns}-popover2-target { margin-top: 0; } diff --git a/web-console/src/components/table-clickable-cell/table-clickable-cell.scss b/web-console/src/components/table-clickable-cell/table-clickable-cell.scss index d6f6f8b2d7f..5c5991df54e 100644 --- a/web-console/src/components/table-clickable-cell/table-clickable-cell.scss +++ b/web-console/src/components/table-clickable-cell/table-clickable-cell.scss @@ -24,6 +24,10 @@ overflow: hidden; text-overflow: ellipsis; + &.disabled { + cursor: not-allowed; + } + .hover-icon { position: absolute; top: $table-cell-v-padding; diff --git a/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx b/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx index cc8cfd71e56..7e4c66fdd5e 100644 --- a/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx +++ b/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx @@ -27,18 +27,23 @@ export interface TableClickableCellProps { onClick: MouseEventHandler; hoverIcon?: IconName; title?: string; + disabled?: boolean; children?: ReactNode; } export const TableClickableCell = React.memo(function TableClickableCell( props: TableClickableCellProps, ) { - const { className, onClick, hoverIcon, title, children } = props; + const { className, onClick, hoverIcon, title, disabled, children } = props; return ( -
+
{children} - {hoverIcon && } + {hoverIcon && !disabled && }
); }); diff --git a/web-console/src/components/warning-checklist/warning-checklist.tsx b/web-console/src/components/warning-checklist/warning-checklist.tsx index 449ad970045..5c74cbdb08a 100644 --- a/web-console/src/components/warning-checklist/warning-checklist.tsx +++ b/web-console/src/components/warning-checklist/warning-checklist.tsx @@ -17,29 +17,31 @@ */ import { Switch } from '@blueprintjs/core'; -import React, { useState } from 'react'; +import React, { ReactNode, useState } from 'react'; export interface WarningChecklistProps { - checks: string[]; - onChange: (allChecked: boolean) => void; + checks: ReactNode[]; + onChange(allChecked: boolean): void; } export const WarningChecklist = React.memo(function WarningChecklist(props: WarningChecklistProps) { const { checks, onChange } = props; - const [checked, setChecked] = useState>({}); + const [checked, setChecked] = useState>({}); - function doCheck(check: string) { + function doCheck(checkIndex: number) { const newChecked = { ...checked }; - newChecked[check] = !newChecked[check]; + newChecked[checkIndex] = !newChecked[checkIndex]; setChecked(newChecked); - onChange(checks.every(check => newChecked[check])); + onChange(checks.every((_, i) => newChecked[i])); } return (
{checks.map((check, i) => ( - doCheck(check)} /> + doCheck(i)}> + {check} + ))}
); diff --git a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx index f892936bab7..0d8cf385a5b 100644 --- a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx +++ b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx @@ -47,7 +47,7 @@ export interface AsyncActionDialogProps { intent?: Intent; successText: string; failText: string; - warningChecks?: string[]; + warningChecks?: ReactNode[]; children?: ReactNode; } diff --git a/web-console/src/dialogs/index.ts b/web-console/src/dialogs/index.ts index 9509442c8bd..588257c84e7 100644 --- a/web-console/src/dialogs/index.ts +++ b/web-console/src/dialogs/index.ts @@ -24,6 +24,7 @@ export * from './diff-dialog/diff-dialog'; export * from './doctor-dialog/doctor-dialog'; export * from './edit-context-dialog/edit-context-dialog'; export * from './history-dialog/history-dialog'; +export * from './kill-datasource-dialog/kill-datasource-dialog'; export * from './lookup-edit-dialog/lookup-edit-dialog'; export * from './numeric-input-dialog/numeric-input-dialog'; export * from './overlord-dynamic-config-dialog/overlord-dynamic-config-dialog'; diff --git a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx new file mode 100644 index 00000000000..3eb7e9fdf24 --- /dev/null +++ b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code, Intent } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +import { FormGroupWithInfo, PopoverText } from '../../components'; +import { SuggestibleInput } from '../../components/suggestible-input/suggestible-input'; +import { Api } from '../../singletons'; +import { uniq } from '../../utils'; +import { AsyncActionDialog } from '../async-action-dialog/async-action-dialog'; + +function getSuggestions(): string[] { + // Default to a data 24h ago so as not to cause a conflict between streaming ingestion and kill tasks + const end = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const startOfDay = end.slice(0, 10); + const startOfMonth = end.slice(0, 7) + '-01'; + const startOfYear = end.slice(0, 4) + '-01-01'; + + return uniq([ + `1000-01-01/${startOfDay}`, + `1000-01-01/${startOfMonth}`, + `1000-01-01/${startOfYear}`, + '1000-01-01/3000-01-01', + ]); +} + +export interface KillDatasourceDialogProps { + datasource: string; + onClose(): void; + onSuccess(): void; +} + +export const KillDatasourceDialog = function KillDatasourceDialog( + props: KillDatasourceDialogProps, +) { + const { datasource, onClose, onSuccess } = props; + const suggestions = getSuggestions(); + const [interval, setInterval] = useState(suggestions[0]); + + return ( + { + const resp = await Api.instance.delete( + `/druid/coordinator/v1/datasources/${Api.encodePath( + datasource, + )}?kill=true&interval=${Api.encodePath(interval)}`, + {}, + ); + return resp.data; + }} + confirmButtonText="Permanently delete unused segments" + successText="Kill task was issued. Unused segments in datasource will be deleted" + failText="Failed submit kill task" + intent={Intent.DANGER} + onClose={onClose} + onSuccess={onSuccess} + warningChecks={[ + <> + I understand that this operation will delete all metadata about the unused segments of{' '} + {datasource} and removes them from deep storage. + , + 'I understand that this operation cannot be undone.', + ]} + > +

+ Are you sure you want to permanently delete unused segments in {datasource}? +

+

This action is not reversible and the data deleted will be lost.

+ +

+ The range of time over which to delete unused segments specified in ISO8601 interval + format. +

+

+ If you have streaming ingestion running make sure that your interval range doe not + overlap with intervals where streaming data is being added - otherwise the kill task + will not start. +

+ + } + > + setInterval(s || '')} + suggestions={suggestions} + /> + + + ); +}; diff --git a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts index 8ed0c514136..9d1254090bf 100644 --- a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts +++ b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts @@ -18,11 +18,7 @@ import { CompactionConfig } from '../compaction-config/compaction-config'; -import { - CompactionStatus, - formatCompactionConfigAndStatus, - zeroCompactionStatus, -} from './compaction-status'; +import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status'; describe('compaction status', () => { const BASIC_CONFIG: CompactionConfig = {}; @@ -61,27 +57,30 @@ describe('compaction status', () => { }); it('formatCompactionConfigAndStatus', () => { - expect(formatCompactionConfigAndStatus(undefined, undefined)).toEqual('Not enabled'); + expect(formatCompactionInfo({})).toEqual('Not enabled'); - expect(formatCompactionConfigAndStatus(BASIC_CONFIG, undefined)).toEqual('Awaiting first run'); + expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run'); - expect(formatCompactionConfigAndStatus(undefined, ZERO_STATUS)).toEqual('Not enabled'); + expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled'); - expect(formatCompactionConfigAndStatus(BASIC_CONFIG, ZERO_STATUS)).toEqual('Running'); + expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running'); expect( - formatCompactionConfigAndStatus(BASIC_CONFIG, { - dataSource: 'tbl', - scheduleStatus: 'RUNNING', - bytesAwaitingCompaction: 0, - bytesCompacted: 100, - bytesSkipped: 0, - segmentCountAwaitingCompaction: 0, - segmentCountCompacted: 10, - segmentCountSkipped: 0, - intervalCountAwaitingCompaction: 0, - intervalCountCompacted: 10, - intervalCountSkipped: 0, + formatCompactionInfo({ + config: BASIC_CONFIG, + status: { + dataSource: 'tbl', + scheduleStatus: 'RUNNING', + bytesAwaitingCompaction: 0, + bytesCompacted: 100, + bytesSkipped: 0, + segmentCountAwaitingCompaction: 0, + segmentCountCompacted: 10, + segmentCountSkipped: 0, + intervalCountAwaitingCompaction: 0, + intervalCountCompacted: 10, + intervalCountSkipped: 0, + }, }), ).toEqual('Fully compacted'); }); diff --git a/web-console/src/druid-models/compaction-status/compaction-status.ts b/web-console/src/druid-models/compaction-status/compaction-status.ts index 2982d9b69e1..d17f2c44fda 100644 --- a/web-console/src/druid-models/compaction-status/compaction-status.ts +++ b/web-console/src/druid-models/compaction-status/compaction-status.ts @@ -50,19 +50,19 @@ export function zeroCompactionStatus(compactionStatus: CompactionStatus): boolea ); } -export function formatCompactionConfigAndStatus( - compactionConfig: CompactionConfig | undefined, - compactionStatus: CompactionStatus | undefined, -) { - if (compactionConfig) { - if (compactionStatus) { - if ( - compactionStatus.bytesAwaitingCompaction === 0 && - !zeroCompactionStatus(compactionStatus) - ) { +export interface CompactionInfo { + config?: CompactionConfig; + status?: CompactionStatus; +} + +export function formatCompactionInfo(compaction: CompactionInfo) { + const { config, status } = compaction; + if (config) { + if (status) { + if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) { return 'Fully compacted'; } else { - return capitalizeFirst(compactionStatus.scheduleStatus); + return capitalizeFirst(status.scheduleStatus); } } else { return 'Awaiting first run'; diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index eeb25db09c4..ca957309ff8 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -69,20 +69,9 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, - { - name: 'killAllDataSources', - type: 'boolean', - defaultValue: false, - info: ( - <> - Send kill tasks for ALL dataSources if property druid.coordinator.kill.on is - true. If this is set to true then killDataSourceWhitelist must not be specified - or be empty list. - - ), - }, { name: 'killDataSourceWhitelist', + label: 'Kill datasource whitelist', type: 'string-array', emptyValue: [], info: ( diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 7f384fc8bc2..c38a42b6393 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -36,12 +36,18 @@ import { TableColumnSelector, ViewControlBar, } from '../../components'; -import { AsyncActionDialog, CompactionDialog, RetentionDialog } from '../../dialogs'; +import { + AsyncActionDialog, + CompactionDialog, + KillDatasourceDialog, + RetentionDialog, +} from '../../dialogs'; import { DatasourceTableActionDialog } from '../../dialogs/datasource-table-action-dialog/datasource-table-action-dialog'; import { CompactionConfig, + CompactionInfo, CompactionStatus, - formatCompactionConfigAndStatus, + formatCompactionInfo, QueryWithContext, zeroCompactionStatus, } from '../../druid-models'; @@ -208,9 +214,8 @@ function segmentGranularityCountsToRank(row: DatasourceQueryResultRow): number { } interface Datasource extends DatasourceQueryResultRow { - readonly rules: Rule[]; - readonly compactionConfig?: CompactionConfig; - readonly compactionStatus?: CompactionStatus; + readonly rules?: Rule[]; + readonly compaction?: CompactionInfo; readonly unused?: boolean; } @@ -220,7 +225,7 @@ function makeUnusedDatasource(datasource: string): Datasource { interface DatasourcesAndDefaultRules { readonly datasources: Datasource[]; - readonly defaultRules: Rule[]; + readonly defaultRules?: Rule[]; } interface RetentionDialogOpenOn { @@ -433,43 +438,85 @@ ORDER BY 1`; let unused: string[] = []; if (showUnused) { - const unusedResp = await Api.instance.get( - '/druid/coordinator/v1/metadata/datasources?includeUnused', - ); - unused = unusedResp.data.filter(d => !seen[d]); + try { + unused = ( + await Api.instance.get( + '/druid/coordinator/v1/metadata/datasources?includeUnused', + ) + ).data.filter(d => !seen[d]); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get the list of unused datasources', + }); + } } - const rulesResp = await Api.instance.get>( - '/druid/coordinator/v1/rules', - ); - const rules = rulesResp.data; + let rules: Record = {}; + try { + rules = (await Api.instance.get>('/druid/coordinator/v1/rules')) + .data; + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get load rules', + }); + } - const compactionConfigsResp = await Api.instance.get<{ - compactionConfigs: CompactionConfig[]; - }>('/druid/coordinator/v1/config/compaction'); - const compactionConfigs = lookupBy( - compactionConfigsResp.data.compactionConfigs || [], - c => c.dataSource, - ); + let compactionConfigs: Record | undefined; + try { + const compactionConfigsResp = await Api.instance.get<{ + compactionConfigs: CompactionConfig[]; + }>('/druid/coordinator/v1/config/compaction'); + compactionConfigs = lookupBy( + compactionConfigsResp.data.compactionConfigs || [], + c => c.dataSource, + ); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get compaction configs', + }); + } - const compactionStatusesResp = await Api.instance.get<{ latestStatus: CompactionStatus[] }>( - '/druid/coordinator/v1/compaction/status', - ); - const compactionStatuses = lookupBy( - compactionStatusesResp.data.latestStatus || [], - c => c.dataSource, - ); + let compactionStatuses: Record | undefined; + if (compactionConfigs) { + // Don't bother getting the statuses if we can not even get the configs + try { + const compactionStatusesResp = await Api.instance.get<{ + latestStatus: CompactionStatus[]; + }>('/druid/coordinator/v1/compaction/status'); + compactionStatuses = lookupBy( + compactionStatusesResp.data.latestStatus || [], + c => c.dataSource, + ); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get compaction statuses', + }); + } + } return { datasources: datasources.concat(unused.map(makeUnusedDatasource)).map(ds => { return { ...ds, - rules: rules[ds.datasource] || [], - compactionConfig: compactionConfigs[ds.datasource], - compactionStatus: compactionStatuses[ds.datasource], + rules: rules[ds.datasource], + compaction: + compactionConfigs && compactionStatuses + ? { + config: compactionConfigs[ds.datasource], + status: compactionStatuses[ds.datasource], + } + : undefined, }; }), - defaultRules: rules[DEFAULT_RULES_KEY] || [], + defaultRules: rules[DEFAULT_RULES_KEY], }; }, onStateChange: datasourcesAndDefaultRulesState => { @@ -633,36 +680,15 @@ ORDER BY 1`; if (!killDatasource) return; return ( - { - const resp = await Api.instance.delete( - `/druid/coordinator/v1/datasources/${Api.encodePath( - killDatasource, - )}?kill=true&interval=1000/3000`, - {}, - ); - return resp.data; - }} - confirmButtonText="Permanently delete unused segments" - successText="Kill task was issued. Unused segments in datasource will be deleted" - failText="Failed submit kill task" - intent={Intent.DANGER} + { this.setState({ killDatasource: undefined }); }} onSuccess={() => { this.fetchDatasourceData(); }} - warningChecks={[ - `I understand that this operation will delete all metadata about the unused segments of ${killDatasource} and removes them from deep storage.`, - 'I understand that this operation cannot be undone.', - ]} - > -

- {`Are you sure you want to permanently delete unused segments in '${killDatasource}'?`} -

-

This action is not reversible and the data deleted will be lost.

- + /> ); } @@ -756,20 +782,20 @@ ORDER BY 1`; this.setState({ retentionDialogOpenOn: undefined }); setTimeout(() => { this.setState(state => { - const datasourcesAndDefaultRules = state.datasourcesAndDefaultRulesState.data; - if (!datasourcesAndDefaultRules) return {}; + const defaultRules = state.datasourcesAndDefaultRulesState.data?.defaultRules; + if (!defaultRules) return {}; return { retentionDialogOpenOn: { datasource: '_default', - rules: datasourcesAndDefaultRules.defaultRules, + rules: defaultRules, }, }; }); }, 50); }; - private readonly saveCompaction = async (compactionConfig: any) => { + private readonly saveCompaction = async (compactionConfig: CompactionConfig) => { if (!compactionConfig) return; try { await Api.instance.post(`/druid/coordinator/v1/config/compaction`, compactionConfig); @@ -819,8 +845,8 @@ ORDER BY 1`; getDatasourceActions( datasource: string, unused: boolean | undefined, - rules: Rule[], - compactionConfig: CompactionConfig | undefined, + rules: Rule[] | undefined, + compactionInfo: CompactionInfo | undefined, ): BasicAction[] { const { goToQuery, goToTask, capabilities } = this.props; @@ -863,82 +889,83 @@ ORDER BY 1`; }, ]; } else { - return goToActions.concat([ - { - icon: IconNames.AUTOMATIC_UPDATES, - title: 'Edit retention rules', - onAction: () => { - this.setState({ - retentionDialogOpenOn: { - datasource, - rules, - }, - }); + return goToActions.concat( + compact([ + { + icon: IconNames.AUTOMATIC_UPDATES, + title: 'Edit retention rules', + onAction: () => { + this.setState({ + retentionDialogOpenOn: { + datasource, + rules: rules || [], + }, + }); + }, }, - }, - { - icon: IconNames.REFRESH, - title: 'Mark as used all segments (will lead to reapplying retention rules)', - onAction: () => - this.setState({ - datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource, - }), - }, - { - icon: IconNames.COMPRESSED, - title: 'Edit compaction configuration', - onAction: () => { - this.setState({ - compactionDialogOpenOn: { - datasource, - compactionConfig, - }, - }); + { + icon: IconNames.REFRESH, + title: 'Mark as used all segments (will lead to reapplying retention rules)', + onAction: () => + this.setState({ + datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource, + }), }, - }, - { - icon: IconNames.EXPORT, - title: 'Mark as used segments by interval', + compactionInfo + ? { + icon: IconNames.COMPRESSED, + title: 'Edit compaction configuration', + onAction: () => { + this.setState({ + compactionDialogOpenOn: { + datasource, + compactionConfig: compactionInfo.config, + }, + }); + }, + } + : undefined, + { + icon: IconNames.EXPORT, + title: 'Mark as used segments by interval', - onAction: () => - this.setState({ - datasourceToMarkSegmentsByIntervalIn: datasource, - useUnuseAction: 'use', - }), - }, - { - icon: IconNames.IMPORT, - title: 'Mark as unused segments by interval', + onAction: () => + this.setState({ + datasourceToMarkSegmentsByIntervalIn: datasource, + useUnuseAction: 'use', + }), + }, + { + icon: IconNames.IMPORT, + title: 'Mark as unused segments by interval', - onAction: () => - this.setState({ - datasourceToMarkSegmentsByIntervalIn: datasource, - useUnuseAction: 'unuse', - }), - }, - { - icon: IconNames.IMPORT, - title: 'Mark as unused all segments', - intent: Intent.DANGER, - onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }), - }, - { - icon: IconNames.TRASH, - title: 'Delete unused segments (issue kill task)', - intent: Intent.DANGER, - onAction: () => this.setState({ killDatasource: datasource }), - }, - ]); + onAction: () => + this.setState({ + datasourceToMarkSegmentsByIntervalIn: datasource, + useUnuseAction: 'unuse', + }), + }, + { + icon: IconNames.IMPORT, + title: 'Mark as unused all segments', + intent: Intent.DANGER, + onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }), + }, + { + icon: IconNames.TRASH, + title: 'Delete unused segments (issue kill task)', + intent: Intent.DANGER, + onAction: () => this.setState({ killDatasource: datasource }), + }, + ]), + ); } } private renderRetentionDialog(): JSX.Element | undefined { const { retentionDialogOpenOn, tiersState, datasourcesAndDefaultRulesState } = this.state; - const { defaultRules } = datasourcesAndDefaultRulesState.data || { - datasources: [], - defaultRules: [], - }; - if (!retentionDialogOpenOn) return; + const defaultRules = datasourcesAndDefaultRulesState.data?.defaultRules; + if (!retentionDialogOpenOn || !defaultRules) return; return ( !d.unused); @@ -1009,8 +1034,8 @@ ORDER BY 1`; const replicatedSizeValues = datasources.map(d => formatReplicatedSize(d.replicated_size)); const leftToBeCompactedValues = datasources.map(d => - d.compactionStatus - ? formatLeftToBeCompacted(d.compactionStatus.bytesAwaitingCompaction) + d.compaction?.status + ? formatLeftToBeCompacted(d.compaction?.status.bytesAwaitingCompaction) : '-', ); @@ -1297,24 +1322,26 @@ ORDER BY 1`; Header: 'Compaction', show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Compaction'), id: 'compactionStatus', - accessor: row => Boolean(row.compactionStatus), + accessor: row => Boolean(row.compaction?.status), filterable: false, width: 150, Cell: ({ original }) => { - const { datasource, compactionConfig, compactionStatus } = original as Datasource; + const { datasource, compaction } = original as Datasource; return ( + disabled={!compaction} + onClick={() => { + if (!compaction) return; this.setState({ compactionDialogOpenOn: { datasource, - compactionConfig, + compactionConfig: compaction.config, }, - }) - } + }); + }} hoverIcon={IconNames.EDIT} > - {formatCompactionConfigAndStatus(compactionConfig, compactionStatus)} + {compaction ? formatCompactionInfo(compaction) : 'Could not get compaction info'} ); }, @@ -1324,17 +1351,22 @@ ORDER BY 1`; show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('% Compacted'), id: 'percentCompacted', width: 200, - accessor: ({ compactionStatus }) => - compactionStatus && compactionStatus.bytesCompacted - ? compactionStatus.bytesCompacted / - (compactionStatus.bytesAwaitingCompaction + compactionStatus.bytesCompacted) - : 0, + accessor: ({ compaction }) => { + const status = compaction?.status; + return status?.bytesCompacted + ? status.bytesCompacted / (status.bytesAwaitingCompaction + status.bytesCompacted) + : 0; + }, filterable: false, className: 'padded', Cell: ({ original }) => { - const { compactionStatus } = original as Datasource; + const { compaction } = original as Datasource; + if (!compaction) { + return 'Could not get compaction info'; + } - if (!compactionStatus || zeroCompactionStatus(compactionStatus)) { + const { status } = compaction; + if (!status || zeroCompactionStatus(status)) { return ( <>  {' '} @@ -1348,10 +1380,14 @@ ORDER BY 1`; <> {' '} +  {' '} + {' '} @@ -1359,18 +1395,8 @@ ORDER BY 1`; {' '} -  {' '} - - (compactionStatus && compactionStatus.bytesAwaitingCompaction) || 0, + accessor: ({ compaction }) => { + const status = compaction?.status; + return status?.bytesAwaitingCompaction || 0; + }, filterable: false, className: 'padded', Cell: ({ original }) => { - const { compactionStatus } = original as Datasource; + const { compaction } = original as Datasource; + if (!compaction) { + return 'Could not get compaction info'; + } - if (!compactionStatus) { + const { status } = compaction; + if (!status) { return ; } return ( ); @@ -1408,26 +1440,30 @@ ORDER BY 1`; Header: 'Retention', show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Retention'), id: 'retention', - accessor: row => row.rules.length, + accessor: row => row.rules?.length || 0, filterable: false, width: 200, Cell: ({ original }) => { const { datasource, rules } = original as Datasource; return ( + disabled={!defaultRules} + onClick={() => { + if (!defaultRules) return; this.setState({ retentionDialogOpenOn: { datasource, - rules, + rules: rules || [], }, - }) - } + }); + }} hoverIcon={IconNames.EDIT} > - {rules.length + {rules?.length ? DatasourcesView.formatRules(rules) - : `Cluster default: ${DatasourcesView.formatRules(defaultRules)}`} + : defaultRules + ? `Cluster default: ${DatasourcesView.formatRules(defaultRules)}` + : 'Could not get default rules'} ); }, @@ -1440,12 +1476,12 @@ ORDER BY 1`; width: ACTION_COLUMN_WIDTH, filterable: false, Cell: ({ value: datasource, original }) => { - const { unused, rules, compactionConfig } = original as Datasource; + const { unused, rules, compaction } = original as Datasource; const datasourceActions = this.getDatasourceActions( datasource, unused, rules, - compactionConfig, + compaction, ); return ( , - Partial {} - export class ServicesView extends React.PureComponent { private readonly serviceQueryManager: QueryManager; @@ -198,7 +196,7 @@ ORDER BY ) DESC, "service" DESC`; - static async getServices(): Promise { + static async getServices(): Promise { const allServiceResp = await Api.instance.get('/druid/coordinator/v1/servers?simple'); const allServices = allServiceResp.data; return allServices.map((s: any) => { @@ -228,7 +226,7 @@ ORDER BY this.serviceQueryManager = new QueryManager({ processQuery: async capabilities => { - let services: ServiceQueryResultRow[]; + let services: ServiceResultRow[]; if (capabilities.hasSql()) { services = await queryDruidSql({ query: ServicesView.SERVICE_SQL }); } else if (capabilities.hasCoordinatorAccess()) { @@ -238,50 +236,49 @@ ORDER BY } if (capabilities.hasCoordinatorAccess()) { - const loadQueueResponse = await Api.instance.get( - '/druid/coordinator/v1/loadqueue?simple', - ); - const loadQueues: Record = loadQueueResponse.data; - services = services.map(s => { - const loadQueueInfo = loadQueues[s.service]; - if (loadQueueInfo) { - s = { ...s, ...loadQueueInfo }; - } - return s; - }); + try { + const loadQueueInfos = ( + await Api.instance.get>( + '/druid/coordinator/v1/loadqueue?simple', + ) + ).data; + services.forEach(s => { + s.loadQueueInfo = loadQueueInfos[s.service]; + }); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'There was an error getting the load queue info', + }); + } } if (capabilities.hasOverlordAccess()) { - let middleManagers: MiddleManagerQueryResultRow[]; try { - const middleManagerResponse = await Api.instance.get('/druid/indexer/v1/workers'); - middleManagers = middleManagerResponse.data; + const workerInfos = (await Api.instance.get('/druid/indexer/v1/workers')) + .data; + + const workerInfoLookup: Record = lookupBy( + workerInfos, + m => m.worker?.host, + ); + + services.forEach(s => { + s.workerInfo = workerInfoLookup[s.service]; + }); } catch (e) { + // Swallow this error because it simply a reflection of a local task runner. if ( - e.response && - typeof e.response.data === 'object' && - e.response.data.error === 'Task Runner does not support worker listing' + deepGet(e, 'response.data.error') !== 'Task Runner does not support worker listing' ) { - // Swallow this error because it simply a reflection of a local task runner. - middleManagers = []; - } else { - // Otherwise re-throw. - throw e; + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'There was an error getting the worker info', + }); } } - - const middleManagersLookup: Record = lookupBy( - middleManagers, - m => m.worker.host, - ); - - services = services.map(s => { - const middleManagerInfo = middleManagersLookup[s.service]; - if (middleManagerInfo) { - s = { ...s, ...middleManagerInfo }; - } - return s; - }); } return services; @@ -372,7 +369,8 @@ ORDER BY id: 'tier', width: 180, accessor: row => { - return row.tier ? row.tier : row.worker ? row.worker.category : null; + if (row.tier) return row.tier; + return deepGet(row, 'workerInfo.worker.category'); }, Cell: this.renderFilterableCell('tier'), }, @@ -451,9 +449,11 @@ ORDER BY className: 'padded', accessor: row => { if (oneOf(row.service_type, 'middle_manager', 'indexer')) { - return row.worker - ? (Number(row.currCapacityUsed) || 0) / Number(row.worker.capacity) - : null; + const { workerInfo } = row; + if (!workerInfo) return 0; + return ( + (Number(workerInfo.currCapacityUsed) || 0) / Number(workerInfo.worker?.capacity) + ); } else { return row.max_size ? Number(row.curr_size) / Number(row.max_size) : null; } @@ -469,15 +469,21 @@ ORDER BY case 'indexer': case 'middle_manager': { - const originalMiddleManagers: ServiceResultRow[] = row.subRows.map( - r => r._original, + const workerInfos: WorkerInfo[] = filterMap( + row.subRows, + r => r._original.workerInfo, ); + + if (!workerInfos.length) { + return 'Could not get worker infos'; + } + const totalCurrCapacityUsed = sum( - originalMiddleManagers, - s => Number(s.currCapacityUsed) || 0, + workerInfos, + w => Number(w.currCapacityUsed) || 0, ); const totalWorkerCapacity = sum( - originalMiddleManagers, + workerInfos, s => deepGet(s, 'worker.capacity') || 0, ); return `${totalCurrCapacityUsed} / ${totalWorkerCapacity} (total slots)`; @@ -496,8 +502,12 @@ ORDER BY case 'indexer': case 'middle_manager': { - const currCapacityUsed = deepGet(row, 'original.currCapacityUsed') || 0; - const capacity = deepGet(row, 'original.worker.capacity'); + if (!deepGet(row, 'original.workerInfo')) { + return 'Could not get capacity info'; + } + const currCapacityUsed = + deepGet(row, 'original.workerInfo.currCapacityUsed') || 0; + const capacity = deepGet(row, 'original.workerInfo.worker.capacity'); if (typeof capacity === 'number') { return `Slots used: ${currCapacityUsed} of ${capacity}`; } else { @@ -518,30 +528,58 @@ ORDER BY filterable: false, className: 'padded', accessor: row => { - if (oneOf(row.service_type, 'middle_manager', 'indexer')) { - if (deepGet(row, 'worker.version') === '') return 'Disabled'; + switch (row.service_type) { + case 'middle_manager': + case 'indexer': { + if (deepGet(row, 'worker.version') === '') return 'Disabled'; + const { workerInfo } = row; + if (!workerInfo) { + return 'Could not get detail info'; + } - const details: string[] = []; - if (row.lastCompletedTaskTime) { - details.push(`Last completed task: ${row.lastCompletedTaskTime}`); + const details: string[] = []; + if (workerInfo.lastCompletedTaskTime) { + details.push(`Last completed task: ${workerInfo.lastCompletedTaskTime}`); + } + if (workerInfo.blacklistedUntil) { + details.push(`Blacklisted until: ${workerInfo.blacklistedUntil}`); + } + return details.join(' '); } - if (row.blacklistedUntil) { - details.push(`Blacklisted until: ${row.blacklistedUntil}`); + + case 'coordinator': + case 'overlord': + return row.is_leader === 1 ? 'Leader' : ''; + + case 'historical': { + const { loadQueueInfo } = row; + if (!loadQueueInfo) return 0; + return ( + (Number(loadQueueInfo.segmentsToLoad) || 0) + + (Number(loadQueueInfo.segmentsToDrop) || 0) + ); } - return details.join(' '); - } else if (oneOf(row.service_type, 'coordinator', 'overlord')) { - return row.is_leader === 1 ? 'Leader' : ''; - } else { - return (Number(row.segmentsToLoad) || 0) + (Number(row.segmentsToDrop) || 0); + + default: + return 0; } }, Cell: row => { if (row.aggregated) return ''; const { service_type } = row.original; switch (service_type) { + case 'middle_manager': + case 'indexer': + case 'coordinator': + case 'overlord': + return row.value; + case 'historical': { + const { loadQueueInfo } = row.original; + if (!loadQueueInfo) return 'Could not get load queue info'; + const { segmentsToLoad, segmentsToLoadSize, segmentsToDrop, segmentsToDropSize } = - row.original; + loadQueueInfo; return formatQueues( segmentsToLoad, segmentsToLoadSize, @@ -550,23 +588,31 @@ ORDER BY ); } - case 'indexer': - case 'middle_manager': - case 'coordinator': - case 'overlord': - return row.value; - default: return ''; } }, Aggregated: row => { if (row.row._pivotVal !== 'historical') return ''; - const originals: ServiceResultRow[] = row.subRows.map(r => r._original); - const segmentsToLoad = sum(originals, s => Number(s.segmentsToLoad) || 0); - const segmentsToLoadSize = sum(originals, s => Number(s.segmentsToLoadSize) || 0); - const segmentsToDrop = sum(originals, s => Number(s.segmentsToDrop) || 0); - const segmentsToDropSize = sum(originals, s => Number(s.segmentsToDropSize) || 0); + const loadQueueInfos: LoadQueueInfo[] = filterMap( + row.subRows, + r => r._original.loadQueueInfo, + ); + + if (!loadQueueInfos.length) { + return 'Could not get load queue infos'; + } + + const segmentsToLoad = sum(loadQueueInfos, s => Number(s.segmentsToLoad) || 0); + const segmentsToLoadSize = sum( + loadQueueInfos, + s => Number(s.segmentsToLoadSize) || 0, + ); + const segmentsToDrop = sum(loadQueueInfos, s => Number(s.segmentsToDrop) || 0); + const segmentsToDropSize = sum( + loadQueueInfos, + s => Number(s.segmentsToDropSize) || 0, + ); return formatQueues( segmentsToLoad, segmentsToLoadSize, @@ -580,13 +626,14 @@ ORDER BY show: capabilities.hasOverlordAccess() && visibleColumns.shown(ACTION_COLUMN_LABEL), id: ACTION_COLUMN_ID, width: ACTION_COLUMN_WIDTH, - accessor: row => row.worker, + accessor: row => row.workerInfo, filterable: false, Cell: ({ value, aggregated }) => { if (aggregated) return ''; if (!value) return null; - const disabled = value.version === ''; - const workerActions = this.getWorkerActions(value.host, disabled); + const { worker } = value; + const disabled = worker.version === ''; + const workerActions = this.getWorkerActions(worker.host, disabled); return ; }, Aggregated: () => '',