Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: be more robust to aux queries failing and improve kill tasks (#13431) 

* be more robust to aux queries failing

* feedback fixes

* remove empty block

* fix spelling

* remove killAllDataSources from the console
This commit is contained in:
Vadim Ogievetsky 2022-11-28 16:50:38 -08:00 committed by GitHub
parent 37b8d4861c
commit d8f4353c43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 527 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
web-console/src/components/auto-form/auto-form.scss
View File

@ -16,19 +16,7 @@
* limitations under the License.
*/
@import '../../variables';
.auto-form {
// Popover in info label
label.#{$bp-ns}-label {
position: relative;
.#{$bp-ns}-text-muted {
position: absolute;
right: 0;
}
}
.custom-input input {
cursor: pointer;
}

14
web-console/src/components/form-group-with-info/form-group-with-info.scss
View File

@ -19,6 +19,20 @@
@import '../../variables';
.form-group-with-info {
label.#{$bp-ns}-label {
position: relative;
.#{$bp-ns}-text-muted {
position: absolute;
right: 0;
// This is only needed because BP4 alerts are too agro in setting CSS on icons
.#{$bp-ns}-icon {
margin-right: 0;
}
}
}
.#{$bp-ns}-text-muted .#{$bp-ns}-popover2-target {
margin-top: 0;
}

4
web-console/src/components/table-clickable-cell/table-clickable-cell.scss
View File

@ -24,6 +24,10 @@
overflow: hidden;
text-overflow: ellipsis;
&.disabled {
cursor: not-allowed;
}
.hover-icon {
position: absolute;
top: $table-cell-v-padding;

11
web-console/src/components/table-clickable-cell/table-clickable-cell.tsx
View File

@ -27,18 +27,23 @@ export interface TableClickableCellProps {
onClick: MouseEventHandler<any>;
hoverIcon?: IconName;
title?: string;
disabled?: boolean;
children?: ReactNode;
}
export const TableClickableCell = React.memo(function TableClickableCell(
props: TableClickableCellProps,
) {
const { className, onClick, hoverIcon, title, children } = props;
const { className, onClick, hoverIcon, title, disabled, children } = props;
return (
<div className={classNames('table-clickable-cell', className)} title={title} onClick={onClick}>
<div
className={classNames('table-clickable-cell', className, { disabled })}
title={title}
onClick={disabled ? undefined : onClick}
>
{children}
{hoverIcon && <Icon className="hover-icon" icon={hoverIcon} />}
{hoverIcon && !disabled && <Icon className="hover-icon" icon={hoverIcon} />}
</div>
);
});

18
web-console/src/components/warning-checklist/warning-checklist.tsx
View File

@ -17,29 +17,31 @@
*/
import { Switch } from '@blueprintjs/core';
import React, { useState } from 'react';
import React, { ReactNode, useState } from 'react';
export interface WarningChecklistProps {
checks: string[];
onChange: (allChecked: boolean) => void;
checks: ReactNode[];
onChange(allChecked: boolean): void;
}
export const WarningChecklist = React.memo(function WarningChecklist(props: WarningChecklistProps) {
const { checks, onChange } = props;
const [checked, setChecked] = useState<Record<string, boolean>>({});
const [checked, setChecked] = useState<Record<number, boolean>>({});
function doCheck(check: string) {
function doCheck(checkIndex: number) {
const newChecked = { ...checked };
newChecked[check] = !newChecked[check];
newChecked[checkIndex] = !newChecked[checkIndex];
setChecked(newChecked);
onChange(checks.every(check => newChecked[check]));
onChange(checks.every((_, i) => newChecked[i]));
}
return (
<div className="warning-checklist">
{checks.map((check, i) => (
<Switch key={i} label={check} onChange={() => doCheck(check)} />
<Switch key={i} onChange={() => doCheck(i)}>
{check}
</Switch>
))}
</div>
);

2
web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx
View File

@ -47,7 +47,7 @@ export interface AsyncActionDialogProps {
intent?: Intent;
successText: string;
failText: string;
warningChecks?: string[];
warningChecks?: ReactNode[];
children?: ReactNode;
}

1
web-console/src/dialogs/index.ts
View File

@ -24,6 +24,7 @@ export * from './diff-dialog/diff-dialog';
export * from './doctor-dialog/doctor-dialog';
export * from './edit-context-dialog/edit-context-dialog';
export * from './history-dialog/history-dialog';
export * from './kill-datasource-dialog/kill-datasource-dialog';
export * from './lookup-edit-dialog/lookup-edit-dialog';
export * from './numeric-input-dialog/numeric-input-dialog';
export * from './overlord-dynamic-config-dialog/overlord-dynamic-config-dialog';

110
web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Code, Intent } from '@blueprintjs/core';
import React, { useState } from 'react';
import { FormGroupWithInfo, PopoverText } from '../../components';
import { SuggestibleInput } from '../../components/suggestible-input/suggestible-input';
import { Api } from '../../singletons';
import { uniq } from '../../utils';
import { AsyncActionDialog } from '../async-action-dialog/async-action-dialog';
function getSuggestions(): string[] {
// Default to a data 24h ago so as not to cause a conflict between streaming ingestion and kill tasks
const end = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
const startOfDay = end.slice(0, 10);
const startOfMonth = end.slice(0, 7) + '-01';
const startOfYear = end.slice(0, 4) + '-01-01';
return uniq([
`1000-01-01/${startOfDay}`,
`1000-01-01/${startOfMonth}`,
`1000-01-01/${startOfYear}`,
'1000-01-01/3000-01-01',
]);
}
export interface KillDatasourceDialogProps {
datasource: string;
onClose(): void;
onSuccess(): void;
}
export const KillDatasourceDialog = function KillDatasourceDialog(
props: KillDatasourceDialogProps,
) {
const { datasource, onClose, onSuccess } = props;
const suggestions = getSuggestions();
const [interval, setInterval] = useState<string>(suggestions[0]);
return (
<AsyncActionDialog
className="kill-datasource-dialog"
action={async () => {
const resp = await Api.instance.delete(
`/druid/coordinator/v1/datasources/${Api.encodePath(
datasource,
)}?kill=true&interval=${Api.encodePath(interval)}`,
{},
);
return resp.data;
}}
confirmButtonText="Permanently delete unused segments"
successText="Kill task was issued. Unused segments in datasource will be deleted"
failText="Failed submit kill task"
intent={Intent.DANGER}
onClose={onClose}
onSuccess={onSuccess}
warningChecks={[
<>
I understand that this operation will delete all metadata about the unused segments of{' '}
<Code>{datasource}</Code> and removes them from deep storage.
</>,
'I understand that this operation cannot be undone.',
]}
>
<p>
Are you sure you want to permanently delete unused segments in <Code>{datasource}</Code>?
</p>
<p>This action is not reversible and the data deleted will be lost.</p>
<FormGroupWithInfo
label="Interval to delete"
info={
<PopoverText>
<p>
The range of time over which to delete unused segments specified in ISO8601 interval
format.
</p>
<p>
If you have streaming ingestion running make sure that your interval range doe not
overlap with intervals where streaming data is being added - otherwise the kill task
will not start.
</p>
</PopoverText>
}
>
<SuggestibleInput
value={interval}
onValueChange={s => setInterval(s || '')}
suggestions={suggestions}
/>
</FormGroupWithInfo>
</AsyncActionDialog>
);
};

41
web-console/src/druid-models/compaction-status/compaction-status.spec.ts
View File

@ -18,11 +18,7 @@
import { CompactionConfig } from '../compaction-config/compaction-config';
import {
CompactionStatus,
formatCompactionConfigAndStatus,
zeroCompactionStatus,
} from './compaction-status';
import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status';
describe('compaction status', () => {
const BASIC_CONFIG: CompactionConfig = {};
@ -61,27 +57,30 @@ describe('compaction status', () => {
});
it('formatCompactionConfigAndStatus', () => {
expect(formatCompactionConfigAndStatus(undefined, undefined)).toEqual('Not enabled');
expect(formatCompactionInfo({})).toEqual('Not enabled');
expect(formatCompactionConfigAndStatus(BASIC_CONFIG, undefined)).toEqual('Awaiting first run');
expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
expect(formatCompactionConfigAndStatus(undefined, ZERO_STATUS)).toEqual('Not enabled');
expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
expect(formatCompactionConfigAndStatus(BASIC_CONFIG, ZERO_STATUS)).toEqual('Running');
expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running');
expect(
formatCompactionConfigAndStatus(BASIC_CONFIG, {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 100,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 10,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 10,
intervalCountSkipped: 0,
formatCompactionInfo({
config: BASIC_CONFIG,
status: {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 100,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 10,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 10,
intervalCountSkipped: 0,
},
}),
).toEqual('Fully compacted');
});

22
web-console/src/druid-models/compaction-status/compaction-status.ts
View File

@ -50,19 +50,19 @@ export function zeroCompactionStatus(compactionStatus: CompactionStatus): boolea
);
}
export function formatCompactionConfigAndStatus(
compactionConfig: CompactionConfig | undefined,
compactionStatus: CompactionStatus | undefined,
) {
if (compactionConfig) {
if (compactionStatus) {
if (
compactionStatus.bytesAwaitingCompaction === 0 &&
!zeroCompactionStatus(compactionStatus)
) {
export interface CompactionInfo {
config?: CompactionConfig;
status?: CompactionStatus;
}
export function formatCompactionInfo(compaction: CompactionInfo) {
const { config, status } = compaction;
if (config) {
if (status) {
if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) {
return 'Fully compacted';
} else {
return capitalizeFirst(compactionStatus.scheduleStatus);
return capitalizeFirst(status.scheduleStatus);
}
} else {
return 'Awaiting first run';

13
web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
View File

@ -69,20 +69,9 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field<CoordinatorDynamicConfig>[
</>
),
},
{
name: 'killAllDataSources',
type: 'boolean',
defaultValue: false,
info: (
<>
Send kill tasks for ALL dataSources if property <Code>druid.coordinator.kill.on</Code> is
true. If this is set to true then <Code>killDataSourceWhitelist</Code> must not be specified
or be empty list.
</>
),
},
{
name: 'killDataSourceWhitelist',
label: 'Kill datasource whitelist',
type: 'string-array',
emptyValue: [],
info: (

396
web-console/src/views/datasources-view/datasources-view.tsx
View File

@ -36,12 +36,18 @@ import {
TableColumnSelector,
ViewControlBar,
} from '../../components';
import { AsyncActionDialog, CompactionDialog, RetentionDialog } from '../../dialogs';
import {
AsyncActionDialog,
CompactionDialog,
KillDatasourceDialog,
RetentionDialog,
} from '../../dialogs';
import { DatasourceTableActionDialog } from '../../dialogs/datasource-table-action-dialog/datasource-table-action-dialog';
import {
CompactionConfig,
CompactionInfo,
CompactionStatus,
formatCompactionConfigAndStatus,
formatCompactionInfo,
QueryWithContext,
zeroCompactionStatus,
} from '../../druid-models';
@ -208,9 +214,8 @@ function segmentGranularityCountsToRank(row: DatasourceQueryResultRow): number {
}
interface Datasource extends DatasourceQueryResultRow {
readonly rules: Rule[];
readonly compactionConfig?: CompactionConfig;
readonly compactionStatus?: CompactionStatus;
readonly rules?: Rule[];
readonly compaction?: CompactionInfo;
readonly unused?: boolean;
}
@ -220,7 +225,7 @@ function makeUnusedDatasource(datasource: string): Datasource {
interface DatasourcesAndDefaultRules {
readonly datasources: Datasource[];
readonly defaultRules: Rule[];
readonly defaultRules?: Rule[];
}
interface RetentionDialogOpenOn {
@ -433,43 +438,85 @@ ORDER BY 1`;
let unused: string[] = [];
if (showUnused) {
const unusedResp = await Api.instance.get<string[]>(
'/druid/coordinator/v1/metadata/datasources?includeUnused',
);
unused = unusedResp.data.filter(d => !seen[d]);
try {
unused = (
await Api.instance.get<string[]>(
'/druid/coordinator/v1/metadata/datasources?includeUnused',
)
).data.filter(d => !seen[d]);
} catch {
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'Could not get the list of unused datasources',
});
}
}
const rulesResp = await Api.instance.get<Record<string, Rule[]>>(
'/druid/coordinator/v1/rules',
);
const rules = rulesResp.data;
let rules: Record<string, Rule[]> = {};
try {
rules = (await Api.instance.get<Record<string, Rule[]>>('/druid/coordinator/v1/rules'))
.data;
} catch {
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'Could not get load rules',
});
}
const compactionConfigsResp = await Api.instance.get<{
compactionConfigs: CompactionConfig[];
}>('/druid/coordinator/v1/config/compaction');
const compactionConfigs = lookupBy(
compactionConfigsResp.data.compactionConfigs || [],
c => c.dataSource,
);
let compactionConfigs: Record<string, CompactionConfig> | undefined;
try {
const compactionConfigsResp = await Api.instance.get<{
compactionConfigs: CompactionConfig[];
}>('/druid/coordinator/v1/config/compaction');
compactionConfigs = lookupBy(
compactionConfigsResp.data.compactionConfigs || [],
c => c.dataSource,
);
} catch {
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'Could not get compaction configs',
});
}
const compactionStatusesResp = await Api.instance.get<{ latestStatus: CompactionStatus[] }>(
'/druid/coordinator/v1/compaction/status',
);
const compactionStatuses = lookupBy(
compactionStatusesResp.data.latestStatus || [],
c => c.dataSource,
);
let compactionStatuses: Record<string, CompactionStatus> | undefined;
if (compactionConfigs) {
// Don't bother getting the statuses if we can not even get the configs
try {
const compactionStatusesResp = await Api.instance.get<{
latestStatus: CompactionStatus[];
}>('/druid/coordinator/v1/compaction/status');
compactionStatuses = lookupBy(
compactionStatusesResp.data.latestStatus || [],
c => c.dataSource,
);
} catch {
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'Could not get compaction statuses',
});
}
}
return {
datasources: datasources.concat(unused.map(makeUnusedDatasource)).map(ds => {
return {
...ds,
rules: rules[ds.datasource] || [],
compactionConfig: compactionConfigs[ds.datasource],
compactionStatus: compactionStatuses[ds.datasource],
rules: rules[ds.datasource],
compaction:
compactionConfigs && compactionStatuses
? {
config: compactionConfigs[ds.datasource],
status: compactionStatuses[ds.datasource],
}
: undefined,
};
}),
defaultRules: rules[DEFAULT_RULES_KEY] || [],
defaultRules: rules[DEFAULT_RULES_KEY],
};
},
onStateChange: datasourcesAndDefaultRulesState => {
@ -633,36 +680,15 @@ ORDER BY 1`;
if (!killDatasource) return;
return (
<AsyncActionDialog
action={async () => {
const resp = await Api.instance.delete(
`/druid/coordinator/v1/datasources/${Api.encodePath(
killDatasource,
)}?kill=true&interval=1000/3000`,
{},
);
return resp.data;
}}
confirmButtonText="Permanently delete unused segments"
successText="Kill task was issued. Unused segments in datasource will be deleted"
failText="Failed submit kill task"
intent={Intent.DANGER}
<KillDatasourceDialog
datasource={killDatasource}
onClose={() => {
this.setState({ killDatasource: undefined });
}}
onSuccess={() => {
this.fetchDatasourceData();
}}
warningChecks={[
`I understand that this operation will delete all metadata about the unused segments of ${killDatasource} and removes them from deep storage.`,
'I understand that this operation cannot be undone.',
]}
>
<p>
{`Are you sure you want to permanently delete unused segments in '${killDatasource}'?`}
</p>
<p>This action is not reversible and the data deleted will be lost.</p>
</AsyncActionDialog>
/>
);
}
@ -756,20 +782,20 @@ ORDER BY 1`;
this.setState({ retentionDialogOpenOn: undefined });
setTimeout(() => {
this.setState(state => {
const datasourcesAndDefaultRules = state.datasourcesAndDefaultRulesState.data;
if (!datasourcesAndDefaultRules) return {};
const defaultRules = state.datasourcesAndDefaultRulesState.data?.defaultRules;
if (!defaultRules) return {};
return {
retentionDialogOpenOn: {
datasource: '_default',
rules: datasourcesAndDefaultRules.defaultRules,
rules: defaultRules,
},
};
});
}, 50);
};
private readonly saveCompaction = async (compactionConfig: any) => {
private readonly saveCompaction = async (compactionConfig: CompactionConfig) => {
if (!compactionConfig) return;
try {
await Api.instance.post(`/druid/coordinator/v1/config/compaction`, compactionConfig);
@ -819,8 +845,8 @@ ORDER BY 1`;
getDatasourceActions(
datasource: string,
unused: boolean | undefined,
rules: Rule[],
compactionConfig: CompactionConfig | undefined,
rules: Rule[] | undefined,
compactionInfo: CompactionInfo | undefined,
): BasicAction[] {
const { goToQuery, goToTask, capabilities } = this.props;
@ -863,82 +889,83 @@ ORDER BY 1`;
},
];
} else {
return goToActions.concat([
{
icon: IconNames.AUTOMATIC_UPDATES,
title: 'Edit retention rules',
onAction: () => {
this.setState({
retentionDialogOpenOn: {
datasource,
rules,
},
});
return goToActions.concat(
compact([
{
icon: IconNames.AUTOMATIC_UPDATES,
title: 'Edit retention rules',
onAction: () => {
this.setState({
retentionDialogOpenOn: {
datasource,
rules: rules || [],
},
});
},
},
},
{
icon: IconNames.REFRESH,
title: 'Mark as used all segments (will lead to reapplying retention rules)',
onAction: () =>
this.setState({
datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource,
}),
},
{
icon: IconNames.COMPRESSED,
title: 'Edit compaction configuration',
onAction: () => {
this.setState({
compactionDialogOpenOn: {
datasource,
compactionConfig,
},
});
{
icon: IconNames.REFRESH,
title: 'Mark as used all segments (will lead to reapplying retention rules)',
onAction: () =>
this.setState({
datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource,
}),
},
},
{
icon: IconNames.EXPORT,
title: 'Mark as used segments by interval',
compactionInfo
? {
icon: IconNames.COMPRESSED,
title: 'Edit compaction configuration',
onAction: () => {
this.setState({
compactionDialogOpenOn: {
datasource,
compactionConfig: compactionInfo.config,
},
});
},
}
: undefined,
{
icon: IconNames.EXPORT,
title: 'Mark as used segments by interval',
onAction: () =>
this.setState({
datasourceToMarkSegmentsByIntervalIn: datasource,
useUnuseAction: 'use',
}),
},
{
icon: IconNames.IMPORT,
title: 'Mark as unused segments by interval',
onAction: () =>
this.setState({
datasourceToMarkSegmentsByIntervalIn: datasource,
useUnuseAction: 'use',
}),
},
{
icon: IconNames.IMPORT,
title: 'Mark as unused segments by interval',
onAction: () =>
this.setState({
datasourceToMarkSegmentsByIntervalIn: datasource,
useUnuseAction: 'unuse',
}),
},
{
icon: IconNames.IMPORT,
title: 'Mark as unused all segments',
intent: Intent.DANGER,
onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }),
},
{
icon: IconNames.TRASH,
title: 'Delete unused segments (issue kill task)',
intent: Intent.DANGER,
onAction: () => this.setState({ killDatasource: datasource }),
},
]);
onAction: () =>
this.setState({
datasourceToMarkSegmentsByIntervalIn: datasource,
useUnuseAction: 'unuse',
}),
},
{
icon: IconNames.IMPORT,
title: 'Mark as unused all segments',
intent: Intent.DANGER,
onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }),
},
{
icon: IconNames.TRASH,
title: 'Delete unused segments (issue kill task)',
intent: Intent.DANGER,
onAction: () => this.setState({ killDatasource: datasource }),
},
]),
);
}
}
private renderRetentionDialog(): JSX.Element | undefined {
const { retentionDialogOpenOn, tiersState, datasourcesAndDefaultRulesState } = this.state;
const { defaultRules } = datasourcesAndDefaultRulesState.data || {
datasources: [],
defaultRules: [],
};
if (!retentionDialogOpenOn) return;
const defaultRules = datasourcesAndDefaultRulesState.data?.defaultRules;
if (!retentionDialogOpenOn || !defaultRules) return;
return (
<RetentionDialog
@ -969,11 +996,11 @@ ORDER BY 1`;
}
private onDetail(datasource: Datasource): void {
const { unused, rules, compactionConfig } = datasource;
const { unused, rules, compaction } = datasource;
this.setState({
datasourceTableActionDialogId: datasource.datasource,
actions: this.getDatasourceActions(datasource.datasource, unused, rules, compactionConfig),
actions: this.getDatasourceActions(datasource.datasource, unused, rules, compaction),
});
}
@ -982,9 +1009,7 @@ ORDER BY 1`;
const { datasourcesAndDefaultRulesState, datasourceFilter, showUnused, visibleColumns } =
this.state;
let { datasources, defaultRules } = datasourcesAndDefaultRulesState.data
? datasourcesAndDefaultRulesState.data
: { datasources: [], defaultRules: [] };
let { datasources, defaultRules } = datasourcesAndDefaultRulesState.data || { datasources: [] };
if (!showUnused) {
datasources = datasources.filter(d => !d.unused);
@ -1009,8 +1034,8 @@ ORDER BY 1`;
const replicatedSizeValues = datasources.map(d => formatReplicatedSize(d.replicated_size));
const leftToBeCompactedValues = datasources.map(d =>
d.compactionStatus
? formatLeftToBeCompacted(d.compactionStatus.bytesAwaitingCompaction)
d.compaction?.status
? formatLeftToBeCompacted(d.compaction?.status.bytesAwaitingCompaction)
: '-',
);
@ -1297,24 +1322,26 @@ ORDER BY 1`;
Header: 'Compaction',
show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Compaction'),
id: 'compactionStatus',
accessor: row => Boolean(row.compactionStatus),
accessor: row => Boolean(row.compaction?.status),
filterable: false,
width: 150,
Cell: ({ original }) => {
const { datasource, compactionConfig, compactionStatus } = original as Datasource;
const { datasource, compaction } = original as Datasource;
return (
<TableClickableCell
onClick={() =>
disabled={!compaction}
onClick={() => {
if (!compaction) return;
this.setState({
compactionDialogOpenOn: {
datasource,
compactionConfig,
compactionConfig: compaction.config,
},
})
}
});
}}
hoverIcon={IconNames.EDIT}
>
{formatCompactionConfigAndStatus(compactionConfig, compactionStatus)}
{compaction ? formatCompactionInfo(compaction) : 'Could not get compaction info'}
</TableClickableCell>
);
},
@ -1324,17 +1351,22 @@ ORDER BY 1`;
show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('% Compacted'),
id: 'percentCompacted',
width: 200,
accessor: ({ compactionStatus }) =>
compactionStatus && compactionStatus.bytesCompacted
? compactionStatus.bytesCompacted /
(compactionStatus.bytesAwaitingCompaction + compactionStatus.bytesCompacted)
: 0,
accessor: ({ compaction }) => {
const status = compaction?.status;
return status?.bytesCompacted
? status.bytesCompacted / (status.bytesAwaitingCompaction + status.bytesCompacted)
: 0;
},
filterable: false,
className: 'padded',
Cell: ({ original }) => {
const { compactionStatus } = original as Datasource;
const { compaction } = original as Datasource;
if (!compaction) {
return 'Could not get compaction info';
}
if (!compactionStatus || zeroCompactionStatus(compactionStatus)) {
const { status } = compaction;
if (!status || zeroCompactionStatus(status)) {
return (
<>
<BracedText text="-" braces={PERCENT_BRACES} /> &nbsp;{' '}
@ -1348,10 +1380,14 @@ ORDER BY 1`;
<>
<BracedText
text={formatPercent(
progress(
compactionStatus.bytesCompacted,
compactionStatus.bytesAwaitingCompaction,
),
progress(status.bytesCompacted, status.bytesAwaitingCompaction),
)}
braces={PERCENT_BRACES}
/>{' '}
&nbsp;{' '}
<BracedText
text={formatPercent(
progress(status.segmentCountCompacted, status.segmentCountAwaitingCompaction),
)}
braces={PERCENT_BRACES}
/>{' '}
@ -1359,18 +1395,8 @@ ORDER BY 1`;
<BracedText
text={formatPercent(
progress(
compactionStatus.segmentCountCompacted,
compactionStatus.segmentCountAwaitingCompaction,
),
)}
braces={PERCENT_BRACES}
/>{' '}
&nbsp;{' '}
<BracedText
text={formatPercent(
progress(
compactionStatus.intervalCountCompacted,
compactionStatus.intervalCountAwaitingCompaction,
status.intervalCountCompacted,
status.intervalCountAwaitingCompaction,
),
)}
braces={PERCENT_BRACES}
@ -1385,20 +1411,26 @@ ORDER BY 1`;
capabilities.hasCoordinatorAccess() && visibleColumns.shown('Left to be compacted'),
id: 'leftToBeCompacted',
width: 100,
accessor: ({ compactionStatus }) =>
(compactionStatus && compactionStatus.bytesAwaitingCompaction) || 0,
accessor: ({ compaction }) => {
const status = compaction?.status;
return status?.bytesAwaitingCompaction || 0;
},
filterable: false,
className: 'padded',
Cell: ({ original }) => {
const { compactionStatus } = original as Datasource;
const { compaction } = original as Datasource;
if (!compaction) {
return 'Could not get compaction info';
}
if (!compactionStatus) {
const { status } = compaction;
if (!status) {
return <BracedText text="-" braces={leftToBeCompactedValues} />;
}
return (
<BracedText
text={formatLeftToBeCompacted(compactionStatus.bytesAwaitingCompaction)}
text={formatLeftToBeCompacted(status.bytesAwaitingCompaction)}
braces={leftToBeCompactedValues}
/>
);
@ -1408,26 +1440,30 @@ ORDER BY 1`;
Header: 'Retention',
show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Retention'),
id: 'retention',
accessor: row => row.rules.length,
accessor: row => row.rules?.length || 0,
filterable: false,
width: 200,
Cell: ({ original }) => {
const { datasource, rules } = original as Datasource;
return (
<TableClickableCell
onClick={() =>
disabled={!defaultRules}
onClick={() => {
if (!defaultRules) return;
this.setState({
retentionDialogOpenOn: {
datasource,
rules,
rules: rules || [],
},
})
}
});
}}
hoverIcon={IconNames.EDIT}
>
{rules.length
{rules?.length
? DatasourcesView.formatRules(rules)
: `Cluster default: ${DatasourcesView.formatRules(defaultRules)}`}
: defaultRules
? `Cluster default: ${DatasourcesView.formatRules(defaultRules)}`
: 'Could not get default rules'}
</TableClickableCell>
);
},
@ -1440,12 +1476,12 @@ ORDER BY 1`;
width: ACTION_COLUMN_WIDTH,
filterable: false,
Cell: ({ value: datasource, original }) => {
const { unused, rules, compactionConfig } = original as Datasource;
const { unused, rules, compaction } = original as Datasource;
const datasourceActions = this.getDatasourceActions(
datasource,
unused,
rules,
compactionConfig,
compaction,
);
return (
<ActionCell

215
web-console/src/views/services-view/services-view.tsx
View File

@ -36,11 +36,12 @@ import {
import { AsyncActionDialog } from '../../dialogs';
import { QueryWithContext } from '../../druid-models';
import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table';
import { Api } from '../../singletons';
import { Api, AppToaster } from '../../singletons';
import {
Capabilities,
CapabilitiesMode,
deepGet,
filterMap,
formatBytes,
formatBytesCompact,
hasPopoverOpen,
@ -117,7 +118,7 @@ export interface ServicesViewState {
visibleColumns: LocalStorageBackedVisibility;
}
interface ServiceQueryResultRow {
interface ServiceResultRow {
readonly service: string;
readonly service_type: string;
readonly tier: string;
@ -127,16 +128,18 @@ interface ServiceQueryResultRow {
readonly max_size: NumberLike;
readonly plaintext_port: number;
readonly tls_port: number;
loadQueueInfo?: LoadQueueInfo;
workerInfo?: WorkerInfo;
}
interface LoadQueueStatus {
interface LoadQueueInfo {
readonly segmentsToDrop: NumberLike;
readonly segmentsToDropSize: NumberLike;
readonly segmentsToLoad: NumberLike;
readonly segmentsToLoadSize: NumberLike;
}
interface MiddleManagerQueryResultRow {
interface WorkerInfo {
readonly availabilityGroups: string[];
readonly blacklistedUntil: string | null;
readonly currCapacityUsed: NumberLike;
@ -153,11 +156,6 @@ interface MiddleManagerQueryResultRow {
};
}
interface ServiceResultRow
extends ServiceQueryResultRow,
Partial<LoadQueueStatus>,
Partial<MiddleManagerQueryResultRow> {}
export class ServicesView extends React.PureComponent<ServicesViewProps, ServicesViewState> {
private readonly serviceQueryManager: QueryManager<Capabilities, ServiceResultRow[]>;
@ -198,7 +196,7 @@ ORDER BY
) DESC,
"service" DESC`;
static async getServices(): Promise<ServiceQueryResultRow[]> {
static async getServices(): Promise<ServiceResultRow[]> {
const allServiceResp = await Api.instance.get('/druid/coordinator/v1/servers?simple');
const allServices = allServiceResp.data;
return allServices.map((s: any) => {
@ -228,7 +226,7 @@ ORDER BY
this.serviceQueryManager = new QueryManager({
processQuery: async capabilities => {
let services: ServiceQueryResultRow[];
let services: ServiceResultRow[];
if (capabilities.hasSql()) {
services = await queryDruidSql({ query: ServicesView.SERVICE_SQL });
} else if (capabilities.hasCoordinatorAccess()) {
@ -238,50 +236,49 @@ ORDER BY
}
if (capabilities.hasCoordinatorAccess()) {
const loadQueueResponse = await Api.instance.get(
'/druid/coordinator/v1/loadqueue?simple',
);
const loadQueues: Record<string, LoadQueueStatus> = loadQueueResponse.data;
services = services.map(s => {
const loadQueueInfo = loadQueues[s.service];
if (loadQueueInfo) {
s = { ...s, ...loadQueueInfo };
}
return s;
});
try {
const loadQueueInfos = (
await Api.instance.get<Record<string, LoadQueueInfo>>(
'/druid/coordinator/v1/loadqueue?simple',
)
).data;
services.forEach(s => {
s.loadQueueInfo = loadQueueInfos[s.service];
});
} catch {
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'There was an error getting the load queue info',
});
}
}
if (capabilities.hasOverlordAccess()) {
let middleManagers: MiddleManagerQueryResultRow[];
try {
const middleManagerResponse = await Api.instance.get('/druid/indexer/v1/workers');
middleManagers = middleManagerResponse.data;
const workerInfos = (await Api.instance.get<WorkerInfo[]>('/druid/indexer/v1/workers'))
.data;
const workerInfoLookup: Record<string, WorkerInfo> = lookupBy(
workerInfos,
m => m.worker?.host,
);
services.forEach(s => {
s.workerInfo = workerInfoLookup[s.service];
});
} catch (e) {
// Swallow this error because it simply a reflection of a local task runner.
if (
e.response &&
typeof e.response.data === 'object' &&
e.response.data.error === 'Task Runner does not support worker listing'
deepGet(e, 'response.data.error') !== 'Task Runner does not support worker listing'
) {
// Swallow this error because it simply a reflection of a local task runner.
middleManagers = [];
} else {
// Otherwise re-throw.
throw e;
AppToaster.show({
icon: IconNames.ERROR,
intent: Intent.DANGER,
message: 'There was an error getting the worker info',
});
}
}
const middleManagersLookup: Record<string, MiddleManagerQueryResultRow> = lookupBy(
middleManagers,
m => m.worker.host,
);
services = services.map(s => {
const middleManagerInfo = middleManagersLookup[s.service];
if (middleManagerInfo) {
s = { ...s, ...middleManagerInfo };
}
return s;
});
}
return services;
@ -372,7 +369,8 @@ ORDER BY
id: 'tier',
width: 180,
accessor: row => {
return row.tier ? row.tier : row.worker ? row.worker.category : null;
if (row.tier) return row.tier;
return deepGet(row, 'workerInfo.worker.category');
},
Cell: this.renderFilterableCell('tier'),
},
@ -451,9 +449,11 @@ ORDER BY
className: 'padded',
accessor: row => {
if (oneOf(row.service_type, 'middle_manager', 'indexer')) {
return row.worker
? (Number(row.currCapacityUsed) || 0) / Number(row.worker.capacity)
: null;
const { workerInfo } = row;
if (!workerInfo) return 0;
return (
(Number(workerInfo.currCapacityUsed) || 0) / Number(workerInfo.worker?.capacity)
);
} else {
return row.max_size ? Number(row.curr_size) / Number(row.max_size) : null;
}
@ -469,15 +469,21 @@ ORDER BY
case 'indexer':
case 'middle_manager': {
const originalMiddleManagers: ServiceResultRow[] = row.subRows.map(
r => r._original,
const workerInfos: WorkerInfo[] = filterMap(
row.subRows,
r => r._original.workerInfo,
);
if (!workerInfos.length) {
return 'Could not get worker infos';
}
const totalCurrCapacityUsed = sum(
originalMiddleManagers,
s => Number(s.currCapacityUsed) || 0,
workerInfos,
w => Number(w.currCapacityUsed) || 0,
);
const totalWorkerCapacity = sum(
originalMiddleManagers,
workerInfos,
s => deepGet(s, 'worker.capacity') || 0,
);
return `${totalCurrCapacityUsed} / ${totalWorkerCapacity} (total slots)`;
@ -496,8 +502,12 @@ ORDER BY
case 'indexer':
case 'middle_manager': {
const currCapacityUsed = deepGet(row, 'original.currCapacityUsed') || 0;
const capacity = deepGet(row, 'original.worker.capacity');
if (!deepGet(row, 'original.workerInfo')) {
return 'Could not get capacity info';
}
const currCapacityUsed =
deepGet(row, 'original.workerInfo.currCapacityUsed') || 0;
const capacity = deepGet(row, 'original.workerInfo.worker.capacity');
if (typeof capacity === 'number') {
return `Slots used: ${currCapacityUsed} of ${capacity}`;
} else {
@ -518,30 +528,58 @@ ORDER BY
filterable: false,
className: 'padded',
accessor: row => {
if (oneOf(row.service_type, 'middle_manager', 'indexer')) {
if (deepGet(row, 'worker.version') === '') return 'Disabled';
switch (row.service_type) {
case 'middle_manager':
case 'indexer': {
if (deepGet(row, 'worker.version') === '') return 'Disabled';
const { workerInfo } = row;
if (!workerInfo) {
return 'Could not get detail info';
}
const details: string[] = [];
if (row.lastCompletedTaskTime) {
details.push(`Last completed task: ${row.lastCompletedTaskTime}`);
const details: string[] = [];
if (workerInfo.lastCompletedTaskTime) {
details.push(`Last completed task: ${workerInfo.lastCompletedTaskTime}`);
}
if (workerInfo.blacklistedUntil) {
details.push(`Blacklisted until: ${workerInfo.blacklistedUntil}`);
}
return details.join(' ');
}
if (row.blacklistedUntil) {
details.push(`Blacklisted until: ${row.blacklistedUntil}`);
case 'coordinator':
case 'overlord':
return row.is_leader === 1 ? 'Leader' : '';
case 'historical': {
const { loadQueueInfo } = row;
if (!loadQueueInfo) return 0;
return (
(Number(loadQueueInfo.segmentsToLoad) || 0) +
(Number(loadQueueInfo.segmentsToDrop) || 0)
);
}
return details.join(' ');
} else if (oneOf(row.service_type, 'coordinator', 'overlord')) {
return row.is_leader === 1 ? 'Leader' : '';
} else {
return (Number(row.segmentsToLoad) || 0) + (Number(row.segmentsToDrop) || 0);
default:
return 0;
}
},
Cell: row => {
if (row.aggregated) return '';
const { service_type } = row.original;
switch (service_type) {
case 'middle_manager':
case 'indexer':
case 'coordinator':
case 'overlord':
return row.value;
case 'historical': {
const { loadQueueInfo } = row.original;
if (!loadQueueInfo) return 'Could not get load queue info';
const { segmentsToLoad, segmentsToLoadSize, segmentsToDrop, segmentsToDropSize } =
row.original;
loadQueueInfo;
return formatQueues(
segmentsToLoad,
segmentsToLoadSize,
@ -550,23 +588,31 @@ ORDER BY
);
}
case 'indexer':
case 'middle_manager':
case 'coordinator':
case 'overlord':
return row.value;
default:
return '';
}
},
Aggregated: row => {
if (row.row._pivotVal !== 'historical') return '';
const originals: ServiceResultRow[] = row.subRows.map(r => r._original);
const segmentsToLoad = sum(originals, s => Number(s.segmentsToLoad) || 0);
const segmentsToLoadSize = sum(originals, s => Number(s.segmentsToLoadSize) || 0);
const segmentsToDrop = sum(originals, s => Number(s.segmentsToDrop) || 0);
const segmentsToDropSize = sum(originals, s => Number(s.segmentsToDropSize) || 0);
const loadQueueInfos: LoadQueueInfo[] = filterMap(
row.subRows,
r => r._original.loadQueueInfo,
);
if (!loadQueueInfos.length) {
return 'Could not get load queue infos';
}
const segmentsToLoad = sum(loadQueueInfos, s => Number(s.segmentsToLoad) || 0);
const segmentsToLoadSize = sum(
loadQueueInfos,
s => Number(s.segmentsToLoadSize) || 0,
);
const segmentsToDrop = sum(loadQueueInfos, s => Number(s.segmentsToDrop) || 0);
const segmentsToDropSize = sum(
loadQueueInfos,
s => Number(s.segmentsToDropSize) || 0,
);
return formatQueues(
segmentsToLoad,
segmentsToLoadSize,
@ -580,13 +626,14 @@ ORDER BY
show: capabilities.hasOverlordAccess() && visibleColumns.shown(ACTION_COLUMN_LABEL),
id: ACTION_COLUMN_ID,
width: ACTION_COLUMN_WIDTH,
accessor: row => row.worker,
accessor: row => row.workerInfo,
filterable: false,
Cell: ({ value, aggregated }) => {
if (aggregated) return '';
if (!value) return null;
const disabled = value.version === '';
const workerActions = this.getWorkerActions(value.host, disabled);
const { worker } = value;
const disabled = worker.version === '';
const workerActions = this.getWorkerActions(worker.host, disabled);
return <ActionCell actions={workerActions} />;
},
Aggregated: () => '',