Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: improve compaction status display (#13523) 

* improve compaction status display

* even more accurate

* fix snapshot
This commit is contained in:
Vadim Ogievetsky 2022-12-07 21:03:59 -08:00 committed by GitHub
parent fbf76ad8f5
commit d85fb8cc4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 181 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
View File

@ -23,6 +23,11 @@
height: 80vh;
}
.legacy-callout {
width: auto;
margin: 10px 15px 0;
}
.form-json-selector {
margin: 15px;
}

27
web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
View File

@ -16,11 +16,16 @@
* limitations under the License.
*/
import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
import { Button, Callout, Classes, Code, Dialog, Intent } from '@blueprintjs/core';
import React, { useState } from 'react';
import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components';
import { COMPACTION_CONFIG_FIELDS, CompactionConfig } from '../../druid-models';
import {
COMPACTION_CONFIG_FIELDS,
CompactionConfig,
compactionConfigHasLegacyInputSegmentSizeBytesSet,
} from '../../druid-models';
import { deepDelete, formatBytesCompact } from '../../utils';
import './compaction-dialog.scss';
@ -55,13 +60,29 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp
canOutsideClickClose={false}
title={`Compaction config: ${datasource}`}
>
{compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && (
<Callout className="legacy-callout" intent={Intent.WARNING}>
<p>
You current config sets the legacy <Code>inputSegmentSizeBytes</Code> to{' '}
<Code>{formatBytesCompact(currentConfig.inputSegmentSizeBytes!)}</Code> it is
recommended to unset this property.
</p>
<p>
<Button
intent={Intent.WARNING}
text="Remove legacy setting"
onClick={() => setCurrentConfig(deepDelete(currentConfig, 'inputSegmentSizeBytes'))}
/>
</p>
</Callout>
)}
<FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
<div className="content">
{currentTab === 'form' ? (
<AutoForm
fields={COMPACTION_CONFIG_FIELDS}
model={currentConfig}
onChange={m => setCurrentConfig(m)}
onChange={m => setCurrentConfig(m as CompactionConfig)}
/>
) : (
<JsonInput

29
web-console/src/druid-models/compaction-config/compaction-config.tsx
View File

@ -22,7 +22,26 @@ import React from 'react';
import { Field } from '../../components';
import { deepGet, deepSet, oneOf } from '../../utils';
export type CompactionConfig = Record<string, any>;
export interface CompactionConfig {
dataSource: string;
skipOffsetFromLatest?: string;
tuningConfig?: any;
[key: string]: any;
// Deprecated:
inputSegmentSizeBytes?: number;
}
export const NOOP_INPUT_SEGMENT_SIZE_BYTES = 100000000000000;
export function compactionConfigHasLegacyInputSegmentSizeBytesSet(
config: CompactionConfig,
): boolean {
return (
typeof config.inputSegmentSizeBytes === 'number' &&
config.inputSegmentSizeBytes < NOOP_INPUT_SEGMENT_SIZE_BYTES
);
}
export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
{
@ -182,7 +201,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
required: (t: CompactionConfig) =>
required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@ -205,7 +224,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
required: (t: CompactionConfig) =>
required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@ -277,7 +296,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
defaultValue: 1073741824,
min: 1000000,
hideInMore: true,
adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of bytes of input segments to process in a single task. If a single segment
@ -293,7 +312,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
defaultValue: 1000,
min: 1,
hideInMore: true,
adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of input segments to process in a single subtask. This limit is to avoid task

146
web-console/src/druid-models/compaction-status/compaction-status.spec.ts
View File

@ -21,7 +21,13 @@ import { CompactionConfig } from '../compaction-config/compaction-config';
import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status';
describe('compaction status', () => {
const BASIC_CONFIG: CompactionConfig = {};
const BASIC_CONFIG: CompactionConfig = {
dataSource: 'tbl',
};
const LEGACY_CONFIG: CompactionConfig = {
dataSource: 'tbl',
inputSegmentSizeBytes: 1e6,
};
const ZERO_STATUS: CompactionStatus = {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
@ -36,52 +42,112 @@ describe('compaction status', () => {
intervalCountSkipped: 0,
};
it('zeroCompactionStatus', () => {
expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
describe('zeroCompactionStatus', () => {
it('works with zero', () => {
expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
});
expect(
zeroCompactionStatus({
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 1,
bytesCompacted: 0,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 0,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 0,
intervalCountSkipped: 0,
}),
).toEqual(false);
});
it('formatCompactionConfigAndStatus', () => {
expect(formatCompactionInfo({})).toEqual('Not enabled');
expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running');
expect(
formatCompactionInfo({
config: BASIC_CONFIG,
status: {
it('works with non-zero', () => {
expect(
zeroCompactionStatus({
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 100,
bytesAwaitingCompaction: 1,
bytesCompacted: 0,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 10,
segmentCountCompacted: 0,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 10,
intervalCountCompacted: 0,
intervalCountSkipped: 0,
},
}),
).toEqual('Fully compacted');
}),
).toEqual(false);
});
});
describe('formatCompactionConfigAndStatus', () => {
it('works with nothing', () => {
expect(formatCompactionInfo({})).toEqual('Not enabled');
});
it('works when there is no status', () => {
expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
});
it('works when here is no config', () => {
expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
});
it('works with config and zero status', () => {
expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual(
'Running',
);
});
it('works when fully compacted', () => {
expect(
formatCompactionInfo({
config: BASIC_CONFIG,
status: {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 100,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 10,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 10,
intervalCountSkipped: 0,
},
}),
).toEqual('Fully compacted');
});
it('works when fully compacted and some segments skipped', () => {
expect(
formatCompactionInfo({
config: BASIC_CONFIG,
status: {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 0,
bytesSkipped: 3776979,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 0,
segmentCountSkipped: 24,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 0,
intervalCountSkipped: 24,
},
}),
).toEqual('Fully compacted (except the last P1D of data, 24 segments skipped)');
});
it('works when fully compacted and some segments skipped (with legacy config)', () => {
expect(
formatCompactionInfo({
config: LEGACY_CONFIG,
status: {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
bytesAwaitingCompaction: 0,
bytesCompacted: 0,
bytesSkipped: 3776979,
segmentCountAwaitingCompaction: 0,
segmentCountCompacted: 0,
segmentCountSkipped: 24,
intervalCountAwaitingCompaction: 0,
intervalCountCompacted: 0,
intervalCountSkipped: 24,
},
}),
).toEqual(
'Fully compacted (except the last P1D of data and segments larger than 1.00MB, 24 segments skipped)',
);
});
});
});

23
web-console/src/druid-models/compaction-status/compaction-status.ts
View File

@ -16,7 +16,11 @@
* limitations under the License.
*/
import { CompactionConfig } from '../compaction-config/compaction-config';
import { formatBytesCompact, pluralIfNeeded } from '../../utils';
import {
CompactionConfig,
compactionConfigHasLegacyInputSegmentSizeBytesSet,
} from '../compaction-config/compaction-config';
function capitalizeFirst(str: string): string {
return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase();
@ -59,8 +63,21 @@ export function formatCompactionInfo(compaction: CompactionInfo) {
const { config, status } = compaction;
if (config) {
if (status) {
if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) {
return 'Fully compacted';
if (
status.bytesAwaitingCompaction === 0 &&
status.segmentCountAwaitingCompaction === 0 &&
status.intervalCountAwaitingCompaction === 0 &&
!zeroCompactionStatus(status)
) {
if (status.segmentCountSkipped) {
return `Fully compacted (except the last ${config.skipOffsetFromLatest || 'P1D'} of data${
compactionConfigHasLegacyInputSegmentSizeBytesSet(config)
? ` and segments larger than ${formatBytesCompact(config.inputSegmentSizeBytes!)}`
: ''
}, ${pluralIfNeeded(status.segmentCountSkipped, 'segment')} skipped)`;
} else {
return 'Fully compacted';
}
} else {
return capitalizeFirst(status.scheduleStatus);
}

2
web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
View File

@ -283,7 +283,7 @@ exports[`DatasourcesView matches snapshot 1`] = `
"filterable": false,
"id": "compactionStatus",
"show": true,
"width": 150,
"width": 180,
},
Object {
"Cell": [Function],

2
web-console/src/views/datasources-view/datasources-view.tsx
View File

@ -1324,7 +1324,7 @@ ORDER BY 1`;
id: 'compactionStatus',
accessor: row => Boolean(row.compaction?.status),
filterable: false,
width: 150,
width: 180,
Cell: ({ original }) => {
const { datasource, compaction } = original as Datasource;
return (