diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
index e3ca37b14ea..499df985c9e 100644
--- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
+++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
@@ -23,6 +23,11 @@
height: 80vh;
}
+ .legacy-callout {
+ width: auto;
+ margin: 10px 15px 0;
+ }
+
.form-json-selector {
margin: 15px;
}
diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
index d63501b1b0b..3b5456e7d04 100644
--- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
+++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
@@ -16,11 +16,16 @@
* limitations under the License.
*/
-import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
+import { Button, Callout, Classes, Code, Dialog, Intent } from '@blueprintjs/core';
import React, { useState } from 'react';
import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components';
-import { COMPACTION_CONFIG_FIELDS, CompactionConfig } from '../../druid-models';
+import {
+ COMPACTION_CONFIG_FIELDS,
+ CompactionConfig,
+ compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../../druid-models';
+import { deepDelete, formatBytesCompact } from '../../utils';
import './compaction-dialog.scss';
@@ -55,13 +60,29 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp
canOutsideClickClose={false}
title={`Compaction config: ${datasource}`}
>
+ {compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && (
+
+
+ You current config sets the legacy inputSegmentSizeBytes to{' '}
+ {formatBytesCompact(currentConfig.inputSegmentSizeBytes!)} it is
+ recommended to unset this property.
+
+
+ setCurrentConfig(deepDelete(currentConfig, 'inputSegmentSizeBytes'))}
+ />
+
+
+ )}
{currentTab === 'form' ? (
setCurrentConfig(m)}
+ onChange={m => setCurrentConfig(m as CompactionConfig)}
/>
) : (
;
+export interface CompactionConfig {
+ dataSource: string;
+ skipOffsetFromLatest?: string;
+ tuningConfig?: any;
+ [key: string]: any;
+
+ // Deprecated:
+ inputSegmentSizeBytes?: number;
+}
+
+export const NOOP_INPUT_SEGMENT_SIZE_BYTES = 100000000000000;
+
+export function compactionConfigHasLegacyInputSegmentSizeBytesSet(
+ config: CompactionConfig,
+): boolean {
+ return (
+ typeof config.inputSegmentSizeBytes === 'number' &&
+ config.inputSegmentSizeBytes < NOOP_INPUT_SEGMENT_SIZE_BYTES
+ );
+}
export const COMPACTION_CONFIG_FIELDS: Field[] = [
{
@@ -182,7 +201,7 @@ export const COMPACTION_CONFIG_FIELDS: Field[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
- required: (t: CompactionConfig) =>
+ required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@@ -205,7 +224,7 @@ export const COMPACTION_CONFIG_FIELDS: Field[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
- required: (t: CompactionConfig) =>
+ required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@@ -277,7 +296,7 @@ export const COMPACTION_CONFIG_FIELDS: Field[] = [
defaultValue: 1073741824,
min: 1000000,
hideInMore: true,
- adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
+ adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of bytes of input segments to process in a single task. If a single segment
@@ -293,7 +312,7 @@ export const COMPACTION_CONFIG_FIELDS: Field[] = [
defaultValue: 1000,
min: 1,
hideInMore: true,
- adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
+ adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of input segments to process in a single subtask. This limit is to avoid task
diff --git a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
index 9d1254090bf..abe660e5c26 100644
--- a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
+++ b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
@@ -21,7 +21,13 @@ import { CompactionConfig } from '../compaction-config/compaction-config';
import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status';
describe('compaction status', () => {
- const BASIC_CONFIG: CompactionConfig = {};
+ const BASIC_CONFIG: CompactionConfig = {
+ dataSource: 'tbl',
+ };
+ const LEGACY_CONFIG: CompactionConfig = {
+ dataSource: 'tbl',
+ inputSegmentSizeBytes: 1e6,
+ };
const ZERO_STATUS: CompactionStatus = {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
@@ -36,52 +42,112 @@ describe('compaction status', () => {
intervalCountSkipped: 0,
};
- it('zeroCompactionStatus', () => {
- expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+ describe('zeroCompactionStatus', () => {
+ it('works with zero', () => {
+ expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+ });
- expect(
- zeroCompactionStatus({
- dataSource: 'tbl',
- scheduleStatus: 'RUNNING',
- bytesAwaitingCompaction: 1,
- bytesCompacted: 0,
- bytesSkipped: 0,
- segmentCountAwaitingCompaction: 0,
- segmentCountCompacted: 0,
- segmentCountSkipped: 0,
- intervalCountAwaitingCompaction: 0,
- intervalCountCompacted: 0,
- intervalCountSkipped: 0,
- }),
- ).toEqual(false);
- });
-
- it('formatCompactionConfigAndStatus', () => {
- expect(formatCompactionInfo({})).toEqual('Not enabled');
-
- expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
-
- expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
-
- expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running');
-
- expect(
- formatCompactionInfo({
- config: BASIC_CONFIG,
- status: {
+ it('works with non-zero', () => {
+ expect(
+ zeroCompactionStatus({
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
- bytesAwaitingCompaction: 0,
- bytesCompacted: 100,
+ bytesAwaitingCompaction: 1,
+ bytesCompacted: 0,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
- segmentCountCompacted: 10,
+ segmentCountCompacted: 0,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
- intervalCountCompacted: 10,
+ intervalCountCompacted: 0,
intervalCountSkipped: 0,
- },
- }),
- ).toEqual('Fully compacted');
+ }),
+ ).toEqual(false);
+ });
+ });
+
+ describe('formatCompactionConfigAndStatus', () => {
+ it('works with nothing', () => {
+ expect(formatCompactionInfo({})).toEqual('Not enabled');
+ });
+
+ it('works when there is no status', () => {
+ expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
+ });
+
+ it('works when here is no config', () => {
+ expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
+ });
+
+ it('works with config and zero status', () => {
+ expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual(
+ 'Running',
+ );
+ });
+
+ it('works when fully compacted', () => {
+ expect(
+ formatCompactionInfo({
+ config: BASIC_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 100,
+ bytesSkipped: 0,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 10,
+ segmentCountSkipped: 0,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 10,
+ intervalCountSkipped: 0,
+ },
+ }),
+ ).toEqual('Fully compacted');
+ });
+
+ it('works when fully compacted and some segments skipped', () => {
+ expect(
+ formatCompactionInfo({
+ config: BASIC_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 0,
+ bytesSkipped: 3776979,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 24,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 24,
+ },
+ }),
+ ).toEqual('Fully compacted (except the last P1D of data, 24 segments skipped)');
+ });
+
+ it('works when fully compacted and some segments skipped (with legacy config)', () => {
+ expect(
+ formatCompactionInfo({
+ config: LEGACY_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 0,
+ bytesSkipped: 3776979,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 24,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 24,
+ },
+ }),
+ ).toEqual(
+ 'Fully compacted (except the last P1D of data and segments larger than 1.00MB, 24 segments skipped)',
+ );
+ });
});
});
diff --git a/web-console/src/druid-models/compaction-status/compaction-status.ts b/web-console/src/druid-models/compaction-status/compaction-status.ts
index d17f2c44fda..c19cb26b6c2 100644
--- a/web-console/src/druid-models/compaction-status/compaction-status.ts
+++ b/web-console/src/druid-models/compaction-status/compaction-status.ts
@@ -16,7 +16,11 @@
* limitations under the License.
*/
-import { CompactionConfig } from '../compaction-config/compaction-config';
+import { formatBytesCompact, pluralIfNeeded } from '../../utils';
+import {
+ CompactionConfig,
+ compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../compaction-config/compaction-config';
function capitalizeFirst(str: string): string {
return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase();
@@ -59,8 +63,21 @@ export function formatCompactionInfo(compaction: CompactionInfo) {
const { config, status } = compaction;
if (config) {
if (status) {
- if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) {
- return 'Fully compacted';
+ if (
+ status.bytesAwaitingCompaction === 0 &&
+ status.segmentCountAwaitingCompaction === 0 &&
+ status.intervalCountAwaitingCompaction === 0 &&
+ !zeroCompactionStatus(status)
+ ) {
+ if (status.segmentCountSkipped) {
+ return `Fully compacted (except the last ${config.skipOffsetFromLatest || 'P1D'} of data${
+ compactionConfigHasLegacyInputSegmentSizeBytesSet(config)
+ ? ` and segments larger than ${formatBytesCompact(config.inputSegmentSizeBytes!)}`
+ : ''
+ }, ${pluralIfNeeded(status.segmentCountSkipped, 'segment')} skipped)`;
+ } else {
+ return 'Fully compacted';
+ }
} else {
return capitalizeFirst(status.scheduleStatus);
}
diff --git a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
index c48713eff20..9bb9c16fef4 100644
--- a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
+++ b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
@@ -283,7 +283,7 @@ exports[`DatasourcesView matches snapshot 1`] = `
"filterable": false,
"id": "compactionStatus",
"show": true,
- "width": 150,
+ "width": 180,
},
Object {
"Cell": [Function],
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx
index c38a42b6393..38b85388d26 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -1324,7 +1324,7 @@ ORDER BY 1`;
id: 'compactionStatus',
accessor: row => Boolean(row.compaction?.status),
filterable: false,
- width: 150,
+ width: 180,
Cell: ({ original }) => {
const { datasource, compaction } = original as Datasource;
return (