Web console: improve compaction status display (#13523)

* improve compaction status display

* even more accurate

* fix snapshot

web-console/src/dialogs/compaction-dialog/compaction-dialog.scss

@@ -23,6 +23,11 @@
     height: 80vh;
   }
 
+  .legacy-callout {
+    width: auto;
+    margin: 10px 15px 0;
+  }
+
   .form-json-selector {
     margin: 15px;
   }

web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx

@@ -16,11 +16,16 @@
  * limitations under the License.
  */
 
-import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
+import { Button, Callout, Classes, Code, Dialog, Intent } from '@blueprintjs/core';
 import React, { useState } from 'react';
 
 import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components';
-import { COMPACTION_CONFIG_FIELDS, CompactionConfig } from '../../druid-models';
+import {
+  COMPACTION_CONFIG_FIELDS,
+  CompactionConfig,
+  compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../../druid-models';
+import { deepDelete, formatBytesCompact } from '../../utils';
 
 import './compaction-dialog.scss';
 
@@ -55,13 +60,29 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp
       canOutsideClickClose={false}
       title={`Compaction config: ${datasource}`}
     >
+      {compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && (
+        <Callout className="legacy-callout" intent={Intent.WARNING}>
+          <p>
+            You current config sets the legacy <Code>inputSegmentSizeBytes</Code> to{' '}
+            <Code>{formatBytesCompact(currentConfig.inputSegmentSizeBytes!)}</Code> it is
+            recommended to unset this property.
+          </p>
+          <p>
+            <Button
+              intent={Intent.WARNING}
+              text="Remove legacy setting"
+              onClick={() => setCurrentConfig(deepDelete(currentConfig, 'inputSegmentSizeBytes'))}
+            />
+          </p>
+        </Callout>
+      )}
       <FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
       <div className="content">
         {currentTab === 'form' ? (
           <AutoForm
             fields={COMPACTION_CONFIG_FIELDS}
             model={currentConfig}
-            onChange={m => setCurrentConfig(m)}
+            onChange={m => setCurrentConfig(m as CompactionConfig)}
           />
         ) : (
           <JsonInput

web-console/src/druid-models/compaction-config/compaction-config.tsx

@@ -22,7 +22,26 @@ import React from 'react';
 import { Field } from '../../components';
 import { deepGet, deepSet, oneOf } from '../../utils';
 
-export type CompactionConfig = Record<string, any>;
+export interface CompactionConfig {
+  dataSource: string;
+  skipOffsetFromLatest?: string;
+  tuningConfig?: any;
+  [key: string]: any;
+
+  // Deprecated:
+  inputSegmentSizeBytes?: number;
+}
+
+export const NOOP_INPUT_SEGMENT_SIZE_BYTES = 100000000000000;
+
+export function compactionConfigHasLegacyInputSegmentSizeBytesSet(
+  config: CompactionConfig,
+): boolean {
+  return (
+    typeof config.inputSegmentSizeBytes === 'number' &&
+    config.inputSegmentSizeBytes < NOOP_INPUT_SEGMENT_SIZE_BYTES
+  );
+}
 
 export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
   {
@@ -182,7 +201,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
     defined: t =>
       oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
       !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
-    required: (t: CompactionConfig) =>
+    required: t =>
       !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
       !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
     info: (
@@ -205,7 +224,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
     defined: t =>
       oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim', 'range') &&
       !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
-    required: (t: CompactionConfig) =>
+    required: t =>
       !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
       !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
     info: (
@@ -277,7 +296,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
     defaultValue: 1073741824,
     min: 1000000,
     hideInMore: true,
-    adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
+    adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
     info: (
       <>
         Maximum number of bytes of input segments to process in a single task. If a single segment
@@ -293,7 +312,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
     defaultValue: 1000,
     min: 1,
     hideInMore: true,
-    adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
+    adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
     info: (
       <>
         Maximum number of input segments to process in a single subtask. This limit is to avoid task

web-console/src/druid-models/compaction-status/compaction-status.spec.ts

@@ -21,7 +21,13 @@ import { CompactionConfig } from '../compaction-config/compaction-config';
 import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status';
 
 describe('compaction status', () => {
-  const BASIC_CONFIG: CompactionConfig = {};
+  const BASIC_CONFIG: CompactionConfig = {
+    dataSource: 'tbl',
+  };
+  const LEGACY_CONFIG: CompactionConfig = {
+    dataSource: 'tbl',
+    inputSegmentSizeBytes: 1e6,
+  };
   const ZERO_STATUS: CompactionStatus = {
     dataSource: 'tbl',
     scheduleStatus: 'RUNNING',
@@ -36,52 +42,112 @@ describe('compaction status', () => {
     intervalCountSkipped: 0,
   };
 
-  it('zeroCompactionStatus', () => {
+  describe('zeroCompactionStatus', () => {
-    expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+    it('works with zero', () => {
+      expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+    });
 
-    expect(
+    it('works with non-zero', () => {
-      zeroCompactionStatus({
+      expect(
-        dataSource: 'tbl',
+        zeroCompactionStatus({
-        scheduleStatus: 'RUNNING',
-        bytesAwaitingCompaction: 1,
-        bytesCompacted: 0,
-        bytesSkipped: 0,
-        segmentCountAwaitingCompaction: 0,
-        segmentCountCompacted: 0,
-        segmentCountSkipped: 0,
-        intervalCountAwaitingCompaction: 0,
-        intervalCountCompacted: 0,
-        intervalCountSkipped: 0,
-      }),
-    ).toEqual(false);
-  });
-
-  it('formatCompactionConfigAndStatus', () => {
-    expect(formatCompactionInfo({})).toEqual('Not enabled');
-
-    expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
-
-    expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
-
-    expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running');
-
-    expect(
-      formatCompactionInfo({
-        config: BASIC_CONFIG,
-        status: {
           dataSource: 'tbl',
           scheduleStatus: 'RUNNING',
-          bytesAwaitingCompaction: 0,
+          bytesAwaitingCompaction: 1,
-          bytesCompacted: 100,
+          bytesCompacted: 0,
           bytesSkipped: 0,
           segmentCountAwaitingCompaction: 0,
-          segmentCountCompacted: 10,
+          segmentCountCompacted: 0,
           segmentCountSkipped: 0,
           intervalCountAwaitingCompaction: 0,
-          intervalCountCompacted: 10,
+          intervalCountCompacted: 0,
           intervalCountSkipped: 0,
-        },
+        }),
-      }),
+      ).toEqual(false);
-    ).toEqual('Fully compacted');
+    });
+  });
+
+  describe('formatCompactionConfigAndStatus', () => {
+    it('works with nothing', () => {
+      expect(formatCompactionInfo({})).toEqual('Not enabled');
+    });
+
+    it('works when there is no status', () => {
+      expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run');
+    });
+
+    it('works when here is no config', () => {
+      expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled');
+    });
+
+    it('works with config and zero status', () => {
+      expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual(
+        'Running',
+      );
+    });
+
+    it('works when fully compacted', () => {
+      expect(
+        formatCompactionInfo({
+          config: BASIC_CONFIG,
+          status: {
+            dataSource: 'tbl',
+            scheduleStatus: 'RUNNING',
+            bytesAwaitingCompaction: 0,
+            bytesCompacted: 100,
+            bytesSkipped: 0,
+            segmentCountAwaitingCompaction: 0,
+            segmentCountCompacted: 10,
+            segmentCountSkipped: 0,
+            intervalCountAwaitingCompaction: 0,
+            intervalCountCompacted: 10,
+            intervalCountSkipped: 0,
+          },
+        }),
+      ).toEqual('Fully compacted');
+    });
+
+    it('works when fully compacted and some segments skipped', () => {
+      expect(
+        formatCompactionInfo({
+          config: BASIC_CONFIG,
+          status: {
+            dataSource: 'tbl',
+            scheduleStatus: 'RUNNING',
+            bytesAwaitingCompaction: 0,
+            bytesCompacted: 0,
+            bytesSkipped: 3776979,
+            segmentCountAwaitingCompaction: 0,
+            segmentCountCompacted: 0,
+            segmentCountSkipped: 24,
+            intervalCountAwaitingCompaction: 0,
+            intervalCountCompacted: 0,
+            intervalCountSkipped: 24,
+          },
+        }),
+      ).toEqual('Fully compacted (except the last P1D of data, 24 segments skipped)');
+    });
+
+    it('works when fully compacted and some segments skipped (with legacy config)', () => {
+      expect(
+        formatCompactionInfo({
+          config: LEGACY_CONFIG,
+          status: {
+            dataSource: 'tbl',
+            scheduleStatus: 'RUNNING',
+            bytesAwaitingCompaction: 0,
+            bytesCompacted: 0,
+            bytesSkipped: 3776979,
+            segmentCountAwaitingCompaction: 0,
+            segmentCountCompacted: 0,
+            segmentCountSkipped: 24,
+            intervalCountAwaitingCompaction: 0,
+            intervalCountCompacted: 0,
+            intervalCountSkipped: 24,
+          },
+        }),
+      ).toEqual(
+        'Fully compacted (except the last P1D of data and segments larger than 1.00MB, 24 segments skipped)',
+      );
+    });
   });
 });

web-console/src/druid-models/compaction-status/compaction-status.ts

@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
-import { CompactionConfig } from '../compaction-config/compaction-config';
+import { formatBytesCompact, pluralIfNeeded } from '../../utils';
+import {
+  CompactionConfig,
+  compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../compaction-config/compaction-config';
 
 function capitalizeFirst(str: string): string {
   return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase();
@@ -59,8 +63,21 @@ export function formatCompactionInfo(compaction: CompactionInfo) {
   const { config, status } = compaction;
   if (config) {
     if (status) {
-      if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) {
+      if (
-        return 'Fully compacted';
+        status.bytesAwaitingCompaction === 0 &&
+        status.segmentCountAwaitingCompaction === 0 &&
+        status.intervalCountAwaitingCompaction === 0 &&
+        !zeroCompactionStatus(status)
+      ) {
+        if (status.segmentCountSkipped) {
+          return `Fully compacted (except the last ${config.skipOffsetFromLatest || 'P1D'} of data${
+            compactionConfigHasLegacyInputSegmentSizeBytesSet(config)
+              ? ` and segments larger than ${formatBytesCompact(config.inputSegmentSizeBytes!)}`
+              : ''
+          }, ${pluralIfNeeded(status.segmentCountSkipped, 'segment')} skipped)`;
+        } else {
+          return 'Fully compacted';
+        }
       } else {
         return capitalizeFirst(status.scheduleStatus);
       }

web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap

@@ -283,7 +283,7 @@ exports[`DatasourcesView matches snapshot 1`] = `
           "filterable": false,
           "id": "compactionStatus",
           "show": true,
-          "width": 150,
+          "width": 180,
         },
         Object {
           "Cell": [Function],

web-console/src/views/datasources-view/datasources-view.tsx

@@ -1324,7 +1324,7 @@ ORDER BY 1`;
             id: 'compactionStatus',
             accessor: row => Boolean(row.compaction?.status),
             filterable: false,
-            width: 150,
+            width: 180,
             Cell: ({ original }) => {
               const { datasource, compaction } = original as Datasource;
               return (