[] = [
name: 'skipOffsetFromLatest',
type: 'string',
defaultValue: 'P1D',
+ suggestions: ['PT0H', 'PT1H', 'P1D', 'P3D'],
info: (
The offset for searching segments to be compacted. Strongly recommended to set for realtime
diff --git a/web-console/src/utils/compaction.spec.ts b/web-console/src/utils/compaction.spec.ts
new file mode 100644
index 00000000000..452cfbe48d6
--- /dev/null
+++ b/web-console/src/utils/compaction.spec.ts
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+ CompactionConfig,
+ CompactionStatus,
+ formatCompactionConfigAndStatus,
+ zeroCompactionStatus,
+} from './compaction';
+
+describe('compaction', () => {
+ const BASIC_CONFIG: CompactionConfig = {};
+ const ZERO_STATUS: CompactionStatus = {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 0,
+ bytesSkipped: 0,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 0,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 0,
+ };
+
+ it('zeroCompactionStatus', () => {
+ expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+
+ expect(
+ zeroCompactionStatus({
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 1,
+ bytesCompacted: 0,
+ bytesSkipped: 0,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 0,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 0,
+ }),
+ ).toEqual(false);
+ });
+
+ it('formatCompactionConfigAndStatus', () => {
+ expect(formatCompactionConfigAndStatus(undefined, undefined)).toEqual('Not enabled');
+
+ expect(formatCompactionConfigAndStatus(BASIC_CONFIG, undefined)).toEqual('Awaiting first run');
+
+ expect(formatCompactionConfigAndStatus(undefined, ZERO_STATUS)).toEqual('Running');
+
+ expect(formatCompactionConfigAndStatus(BASIC_CONFIG, ZERO_STATUS)).toEqual('Running');
+
+ expect(
+ formatCompactionConfigAndStatus(BASIC_CONFIG, {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 100,
+ bytesSkipped: 0,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 10,
+ segmentCountSkipped: 0,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 10,
+ intervalCountSkipped: 0,
+ }),
+ ).toEqual('Fully compacted');
+ });
+});
diff --git a/web-console/src/utils/compaction.ts b/web-console/src/utils/compaction.ts
new file mode 100644
index 00000000000..34634a1f9b7
--- /dev/null
+++ b/web-console/src/utils/compaction.ts
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function capitalizeFirst(str: string): string {
+ return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase();
+}
+
+export interface CompactionStatus {
+ dataSource: string;
+ scheduleStatus: string;
+ bytesAwaitingCompaction: number;
+ bytesCompacted: number;
+ bytesSkipped: number;
+ segmentCountAwaitingCompaction: number;
+ segmentCountCompacted: number;
+ segmentCountSkipped: number;
+ intervalCountAwaitingCompaction: number;
+ intervalCountCompacted: number;
+ intervalCountSkipped: number;
+}
+
+export type CompactionConfig = Record;
+
+export function zeroCompactionStatus(compactionStatus: CompactionStatus): boolean {
+ return (
+ !compactionStatus.bytesAwaitingCompaction &&
+ !compactionStatus.bytesCompacted &&
+ !compactionStatus.bytesSkipped &&
+ !compactionStatus.segmentCountAwaitingCompaction &&
+ !compactionStatus.segmentCountCompacted &&
+ !compactionStatus.segmentCountSkipped &&
+ !compactionStatus.intervalCountAwaitingCompaction &&
+ !compactionStatus.intervalCountCompacted &&
+ !compactionStatus.intervalCountSkipped
+ );
+}
+
+export function formatCompactionConfigAndStatus(
+ compactionConfig: CompactionConfig | undefined,
+ compactionStatus: CompactionStatus | undefined,
+) {
+ if (compactionStatus) {
+ if (compactionStatus.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(compactionStatus)) {
+ return 'Fully compacted';
+ } else {
+ return capitalizeFirst(compactionStatus.scheduleStatus);
+ }
+ } else if (compactionConfig) {
+ return 'Awaiting first run';
+ } else {
+ return 'Not enabled';
+ }
+}
diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts
index 2a327bdb3f2..9b2398bb9ce 100644
--- a/web-console/src/utils/general.spec.ts
+++ b/web-console/src/utils/general.spec.ts
@@ -18,6 +18,11 @@
import {
alphanumericCompare,
+ formatBytes,
+ formatBytesCompact,
+ formatInteger,
+ formatMegabytes,
+ formatPercent,
sortWithPrefixSuffix,
sqlQueryCustomTableFilter,
swapElements,
@@ -83,4 +88,34 @@ describe('general', () => {
expect(swapElements(array, 2, 4)).toEqual(['a', 'b', 'e', 'd', 'c']);
});
});
+
+ describe('formatInteger', () => {
+ it('works', () => {
+ expect(formatInteger(10000)).toEqual('10,000');
+ });
+ });
+
+ describe('formatBytes', () => {
+ it('works', () => {
+ expect(formatBytes(10000)).toEqual('10.00 KB');
+ });
+ });
+
+ describe('formatBytesCompact', () => {
+ it('works', () => {
+ expect(formatBytesCompact(10000)).toEqual('10.00KB');
+ });
+ });
+
+ describe('formatMegabytes', () => {
+ it('works', () => {
+ expect(formatMegabytes(30000000)).toEqual('28.6');
+ });
+ });
+
+ describe('formatPercent', () => {
+ it('works', () => {
+ expect(formatPercent(2 / 3)).toEqual('66.67%');
+ });
+ });
});
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 47c7e17c936..7afe3856303 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -231,6 +231,10 @@ export function formatMegabytes(n: number): string {
return numeral(n / 1048576).format('0,0.0');
}
+export function formatPercent(n: number): string {
+ return (n * 100).toFixed(2) + '%';
+}
+
function pad2(str: string | number): string {
return ('00' + str).substr(-2);
}
diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx
index b46d675819f..2bcf661c7f9 100644
--- a/web-console/src/utils/index.tsx
+++ b/web-console/src/utils/index.tsx
@@ -24,3 +24,4 @@ export * from './query-manager';
export * from './query-cursor';
export * from './local-storage-keys';
export * from './column-metadata';
+export * from './compaction';
diff --git a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
index 1d5ce02941f..2364a8a2769 100755
--- a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
+++ b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
@@ -11,7 +11,20 @@ exports[`data source view matches snapshot 1`] = `
localStorageKey="datasources-refresh-rate"
onRefresh={[Function]}
/>
-
+
+ }
+ >
+ % Compacted
+
+ bytes / segments / intervals
+ ,
+ "accessor": [Function],
+ "filterable": false,
+ "id": "percentCompacted",
+ "show": true,
+ "width": 200,
+ },
+ Object {
+ "Cell": [Function],
+ "Header":
+ Left to be
+
+ compacted
+ ,
+ "accessor": [Function],
+ "filterable": false,
+ "id": "leftToBeCompacted",
+ "show": true,
+ "width": 100,
},
Object {
"Cell": [Function],
@@ -226,6 +270,7 @@ exports[`data source view matches snapshot 1`] = `
"accessor": [Function],
"filterable": false,
"id": "retention",
+ "minWidth": 100,
"show": true,
},
Object {
diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx
index 20a38e73aee..59688b04a65 100644
--- a/web-console/src/views/datasource-view/datasource-view.tsx
+++ b/web-console/src/views/datasource-view/datasource-view.tsx
@@ -37,20 +37,19 @@ import {
TableColumnSelector,
ViewControlBar,
} from '../../components';
-import {
- AsyncActionDialog,
- CompactionDialog,
- DEFAULT_MAX_ROWS_PER_SEGMENT,
- RetentionDialog,
-} from '../../dialogs';
+import { AsyncActionDialog, CompactionDialog, RetentionDialog } from '../../dialogs';
import { DatasourceTableActionDialog } from '../../dialogs/datasource-table-action-dialog/datasource-table-action-dialog';
import { AppToaster } from '../../singletons/toaster';
import {
addFilter,
+ CompactionConfig,
+ CompactionStatus,
countBy,
formatBytes,
+ formatCompactionConfigAndStatus,
formatInteger,
formatMegabytes,
+ formatPercent,
getDruidErrorMessage,
LocalStorageKeys,
lookupBy,
@@ -58,6 +57,7 @@ import {
queryDruidSql,
QueryManager,
QueryState,
+ zeroCompactionStatus,
} from '../../utils';
import { BasicAction } from '../../utils/basic-action';
import { Capabilities, CapabilitiesMode } from '../../utils/capabilities';
@@ -78,6 +78,8 @@ const tableColumns: Record = {
'Avg. row size',
'Replicated size',
'Compaction',
+ '% Compacted',
+ 'Left to be compacted',
'Retention',
ACTION_COLUMN_LABEL,
],
@@ -88,6 +90,8 @@ const tableColumns: Record = {
'Total data size',
'Segment size',
'Compaction',
+ '% Compacted',
+ 'Left to be compacted',
'Retention',
ACTION_COLUMN_LABEL,
],
@@ -107,10 +111,10 @@ const tableColumns: Record = {
function formatLoadDrop(segmentsToLoad: number, segmentsToDrop: number): string {
const loadDrop: string[] = [];
if (segmentsToLoad) {
- loadDrop.push(`${segmentsToLoad} segments to load`);
+ loadDrop.push(`${pluralIfNeeded(segmentsToLoad, 'segment')} to load`);
}
if (segmentsToDrop) {
- loadDrop.push(`${segmentsToDrop} segments to drop`);
+ loadDrop.push(`${pluralIfNeeded(segmentsToDrop, 'segment')} to drop`);
}
return loadDrop.join(', ') || 'No segments to load/drop';
}
@@ -120,6 +124,7 @@ const formatSegmentSize = formatMegabytes;
const formatTotalRows = formatInteger;
const formatAvgRowSize = formatInteger;
const formatReplicatedSize = formatBytes;
+const formatLeftToBeCompacted = formatBytes;
function twoLines(line1: string, line2: string) {
return (
@@ -131,9 +136,19 @@ function twoLines(line1: string, line2: string) {
);
}
+function progress(done: number, awaiting: number): number {
+ const d = done + awaiting;
+ if (!d) return 0;
+ return done / d;
+}
+
+const PERCENT_BRACES = [formatPercent(1)];
+
interface Datasource {
datasource: string;
rules: Rule[];
+ compactionConfig?: CompactionConfig;
+ compactionStatus?: CompactionStatus;
[key: string]: any;
}
@@ -164,7 +179,7 @@ interface RetentionDialogOpenOn {
interface CompactionDialogOpenOn {
datasource: string;
- compactionConfig: Record;
+ compactionConfig: CompactionConfig;
}
export interface DatasourcesViewProps {
@@ -190,6 +205,7 @@ export interface DatasourcesViewState {
datasourceToMarkSegmentsByIntervalIn?: string;
useUnuseAction: 'use' | 'unuse';
useUnuseInterval: string;
+ showForceCompact: boolean;
hiddenColumns: LocalStorageBackedArray;
showChart: boolean;
chartWidth: number;
@@ -229,7 +245,7 @@ export class DatasourcesView extends React.PureComponent<
FROM sys.segments
GROUP BY 1`;
- static formatRules(rules: any[]): string {
+ static formatRules(rules: Rule[]): string {
if (rules.length === 0) {
return 'No rules';
} else if (rules.length <= 2) {
@@ -259,6 +275,7 @@ GROUP BY 1`;
showUnused: false,
useUnuseAction: 'unuse',
useUnuseInterval: '',
+ showForceCompact: false,
hiddenColumns: new LocalStorageBackedArray(
LocalStorageKeys.DATASOURCE_TABLE_COLUMN_SELECTION,
),
@@ -314,7 +331,7 @@ GROUP BY 1`;
};
}
- const seen = countBy(datasources, (x: any) => x.datasource);
+ const seen = countBy(datasources, x => x.datasource);
let unused: string[] = [];
if (this.state.showUnused) {
@@ -329,18 +346,25 @@ GROUP BY 1`;
const rulesResp = await axios.get('/druid/coordinator/v1/rules');
const rules = rulesResp.data;
- const compactionResp = await axios.get('/druid/coordinator/v1/config/compaction');
- const compaction = lookupBy(
- compactionResp.data.compactionConfigs,
- (c: any) => c.dataSource,
+ const compactionConfigsResp = await axios.get('/druid/coordinator/v1/config/compaction');
+ const compactionConfigs = lookupBy(
+ compactionConfigsResp.data.compactionConfigs || [],
+ (c: CompactionConfig) => c.dataSource,
+ );
+
+ const compactionStatusesResp = await axios.get('/druid/coordinator/v1/compaction/status');
+ const compactionStatuses = lookupBy(
+ compactionStatusesResp.data.latestStatus || [],
+ (c: CompactionStatus) => c.dataSource,
);
const allDatasources = (datasources as any).concat(
unused.map(d => ({ datasource: d, unused: true })),
);
- allDatasources.forEach((ds: any) => {
+ allDatasources.forEach((ds: Datasource) => {
ds.rules = rules[ds.datasource] || [];
- ds.compaction = compaction[ds.datasource];
+ ds.compactionConfig = compactionConfigs[ds.datasource];
+ ds.compactionStatus = compactionStatuses[ds.datasource];
});
return {
@@ -535,7 +559,18 @@ GROUP BY 1`;
const { goToQuery, capabilities } = this.props;
return (
-
+ {
+ this.setState({ showForceCompact: true });
+ }}
+ />
+ }
+ >
{capabilities.hasSql() && (