Web console: Index spec dialog (#13425)

* add index spec dialog

* add sanpshot

web-console/src/components/auto-form/auto-form.scss

@@ -28,4 +28,8 @@
       right: 0;
     }
   }
+
+  .custom-input input {
+    cursor: pointer;
+  }
 }

web-console/src/components/auto-form/auto-form.tsx

@@ -16,7 +16,14 @@
  * limitations under the License.
  */
 
-import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core';
+import {
+  Button,
+  ButtonGroup,
+  FormGroup,
+  InputGroup,
+  Intent,
+  NumericInput,
+} from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import React from 'react';
 
@@ -46,7 +53,8 @@ export interface Field<M> {
     | 'boolean'
     | 'string-array'
     | 'json'
-    | 'interval';
+    | 'interval'
+    | 'custom';
   defaultValue?: any;
   emptyValue?: any;
   suggestions?: Functor<M, Suggestion[]>;
@@ -64,6 +72,13 @@ export interface Field<M> {
   valueAdjustment?: (value: any) => any;
   adjustment?: (model: Partial<M>) => Partial<M>;
   issueWithValue?: (value: any) => string | undefined;
+
+  customSummary?: (v: any) => string;
+  customDialog?: (o: {
+    value: any;
+    onValueChange: (v: any) => void;
+    onClose: () => void;
+  }) => JSX.Element;
 }
 
 interface ComputedFieldValues {
@@ -84,6 +99,7 @@ export interface AutoFormProps<M> {
 
 export interface AutoFormState {
   showMore: boolean;
+  customDialog?: JSX.Element;
 }
 
 export class AutoForm<T extends Record<string, any>> extends React.PureComponent<
@@ -395,6 +411,36 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     );
   }
 
+  private renderCustomInput(field: Field<T>): JSX.Element {
+    const { model } = this.props;
+    const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field);
+    const effectiveValue = modelValue || defaultValue;
+
+    const onEdit = () => {
+      this.setState({
+        customDialog: field.customDialog?.({
+          value: effectiveValue,
+          onValueChange: v => this.fieldChange(field, v),
+          onClose: () => {
+            this.setState({ customDialog: undefined });
+          },
+        }),
+      });
+    };
+
+    return (
+      <InputGroup
+        className="custom-input"
+        value={(field.customSummary || String)(effectiveValue)}
+        intent={required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined}
+        readOnly
+        placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
+        rightElement={<Button icon={IconNames.EDIT} minimal onClick={onEdit} />}
+        onClick={onEdit}
+      />
+    );
+  }
+
   renderFieldInput(field: Field<T>) {
     switch (field.type) {
       case 'number':
@@ -413,6 +459,8 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         return this.renderJsonInput(field);
       case 'interval':
         return this.renderIntervalInput(field);
+      case 'custom':
+        return this.renderCustomInput(field);
       default:
         throw new Error(`unknown field type '${field.type}'`);
     }
@@ -464,7 +512,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
 
   render(): JSX.Element {
     const { fields, model, showCustom } = this.props;
-    const { showMore } = this.state;
+    const { showMore, customDialog } = this.state;
 
     let shouldShowMore = false;
     const shownFields = fields.filter(field => {
@@ -489,6 +537,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         {model && shownFields.map(this.renderField)}
         {model && showCustom && showCustom(model) && this.renderCustom()}
         {shouldShowMore && this.renderMoreOrLess()}
+        {customDialog}
       </div>
     );
   }

web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap

@@ -0,0 +1,317 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`IndexSpecDialog matches snapshot with indexSpec 1`] = `
+<Blueprint4.Dialog
+  canOutsideClickClose={false}
+  className="index-spec-dialog"
+  isOpen={true}
+  onClose={[Function]}
+  title="Index spec"
+>
+  <Memo(FormJsonSelector)
+    onChange={[Function]}
+    tab="form"
+  />
+  <div
+    className="content"
+  >
+    <AutoForm
+      fields={
+        Array [
+          Object {
+            "defaultValue": "utf8",
+            "info": <React.Fragment>
+              Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns.
+            </React.Fragment>,
+            "label": "String dictionary encoding",
+            "name": "stringDictionaryEncoding.type",
+            "suggestions": Array [
+              "utf8",
+              "frontCoded",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": 4,
+            "defined": [Function],
+            "info": <React.Fragment>
+              The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128.
+            </React.Fragment>,
+            "label": "String dictionary encoding bucket size",
+            "max": 128,
+            "min": 1,
+            "name": "stringDictionaryEncoding.bucketSize",
+            "type": "number",
+          },
+          Object {
+            "defaultValue": "roaring",
+            "info": <React.Fragment>
+              Compression format for bitmap indexes.
+            </React.Fragment>,
+            "label": "Bitmap type",
+            "name": "bitmap.type",
+            "suggestions": Array [
+              "roaring",
+              "concise",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": true,
+            "defined": [Function],
+            "info": <React.Fragment>
+              Controls whether or not run-length encoding will be used when it is determined to be more space-efficient.
+            </React.Fragment>,
+            "label": "Bitmap compress run on serialization",
+            "name": "bitmap.compressRunOnSerialization",
+            "type": "boolean",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format for dimension columns.
+            </React.Fragment>,
+            "name": "dimensionCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "longs",
+            "info": <React.Fragment>
+              Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. 
+              <Unknown>
+                auto
+              </Unknown>
+               encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. 
+              <Unknown>
+                longs
+              </Unknown>
+               stores the value as-is with 8 bytes each.
+            </React.Fragment>,
+            "name": "longEncoding",
+            "suggestions": Array [
+              "longs",
+              "auto",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format for primitive type metric columns.
+            </React.Fragment>,
+            "name": "metricCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format to use for nested column raw data. 
+            </React.Fragment>,
+            "label": "JSON compression",
+            "name": "jsonCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+        ]
+      }
+      model={
+        Object {
+          "dimensionCompression": "lzf",
+        }
+      }
+      onChange={[Function]}
+    />
+  </div>
+  <div
+    className="bp4-dialog-footer"
+  >
+    <div
+      className="bp4-dialog-footer-actions"
+    >
+      <Blueprint4.Button
+        onClick={[Function]}
+        text="Close"
+      />
+      <Blueprint4.Button
+        disabled={false}
+        intent="primary"
+        onClick={[Function]}
+        text="Save"
+      />
+    </div>
+  </div>
+</Blueprint4.Dialog>
+`;
+
+exports[`IndexSpecDialog matches snapshot without compactionConfig 1`] = `
+<Blueprint4.Dialog
+  canOutsideClickClose={false}
+  className="index-spec-dialog"
+  isOpen={true}
+  onClose={[Function]}
+  title="Index spec"
+>
+  <Memo(FormJsonSelector)
+    onChange={[Function]}
+    tab="form"
+  />
+  <div
+    className="content"
+  >
+    <AutoForm
+      fields={
+        Array [
+          Object {
+            "defaultValue": "utf8",
+            "info": <React.Fragment>
+              Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns.
+            </React.Fragment>,
+            "label": "String dictionary encoding",
+            "name": "stringDictionaryEncoding.type",
+            "suggestions": Array [
+              "utf8",
+              "frontCoded",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": 4,
+            "defined": [Function],
+            "info": <React.Fragment>
+              The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128.
+            </React.Fragment>,
+            "label": "String dictionary encoding bucket size",
+            "max": 128,
+            "min": 1,
+            "name": "stringDictionaryEncoding.bucketSize",
+            "type": "number",
+          },
+          Object {
+            "defaultValue": "roaring",
+            "info": <React.Fragment>
+              Compression format for bitmap indexes.
+            </React.Fragment>,
+            "label": "Bitmap type",
+            "name": "bitmap.type",
+            "suggestions": Array [
+              "roaring",
+              "concise",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": true,
+            "defined": [Function],
+            "info": <React.Fragment>
+              Controls whether or not run-length encoding will be used when it is determined to be more space-efficient.
+            </React.Fragment>,
+            "label": "Bitmap compress run on serialization",
+            "name": "bitmap.compressRunOnSerialization",
+            "type": "boolean",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format for dimension columns.
+            </React.Fragment>,
+            "name": "dimensionCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "longs",
+            "info": <React.Fragment>
+              Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. 
+              <Unknown>
+                auto
+              </Unknown>
+               encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. 
+              <Unknown>
+                longs
+              </Unknown>
+               stores the value as-is with 8 bytes each.
+            </React.Fragment>,
+            "name": "longEncoding",
+            "suggestions": Array [
+              "longs",
+              "auto",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format for primitive type metric columns.
+            </React.Fragment>,
+            "name": "metricCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+          Object {
+            "defaultValue": "lz4",
+            "info": <React.Fragment>
+              Compression format to use for nested column raw data. 
+            </React.Fragment>,
+            "label": "JSON compression",
+            "name": "jsonCompression",
+            "suggestions": Array [
+              "lz4",
+              "lzf",
+              "zstd",
+              "uncompressed",
+            ],
+            "type": "string",
+          },
+        ]
+      }
+      model={Object {}}
+      onChange={[Function]}
+    />
+  </div>
+  <div
+    className="bp4-dialog-footer"
+  >
+    <div
+      className="bp4-dialog-footer-actions"
+    >
+      <Blueprint4.Button
+        onClick={[Function]}
+        text="Close"
+      />
+      <Blueprint4.Button
+        disabled={false}
+        intent="primary"
+        onClick={[Function]}
+        text="Save"
+      />
+    </div>
+  </div>
+</Blueprint4.Dialog>
+`;

web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../variables';
+
+.index-spec-dialog {
+  &.#{$bp-ns}-dialog {
+    height: 70vh;
+  }
+
+  .form-json-selector {
+    margin: 15px;
+  }
+
+  .content {
+    margin: 0 15px 10px 0;
+    padding: 0 5px 0 15px;
+    flex: 1;
+    overflow: auto;
+  }
+}

web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { shallow } from 'enzyme';
+import React from 'react';
+
+import { IndexSpecDialog } from './index-spec-dialog';
+
+describe('IndexSpecDialog', () => {
+  it('matches snapshot without compactionConfig', () => {
+    const compactionDialog = shallow(
+      <IndexSpecDialog onClose={() => {}} onSave={() => {}} indexSpec={undefined} />,
+    );
+    expect(compactionDialog).toMatchSnapshot();
+  });
+
+  it('matches snapshot with indexSpec', () => {
+    const compactionDialog = shallow(
+      <IndexSpecDialog
+        onClose={() => {}}
+        onSave={() => {}}
+        indexSpec={{
+          dimensionCompression: 'lzf',
+        }}
+      />,
+    );
+    expect(compactionDialog).toMatchSnapshot();
+  });
+});

web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
+import React, { useState } from 'react';
+
+import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components';
+import { INDEX_SPEC_FIELDS, IndexSpec } from '../../druid-models';
+
+import './index-spec-dialog.scss';
+
+export interface IndexSpecDialogProps {
+  title?: string;
+  onClose: () => void;
+  onSave: (indexSpec: IndexSpec) => void;
+  indexSpec: IndexSpec | undefined;
+}
+
+export const IndexSpecDialog = React.memo(function IndexSpecDialog(props: IndexSpecDialogProps) {
+  const { title, indexSpec, onSave, onClose } = props;
+
+  const [currentTab, setCurrentTab] = useState<FormJsonTabs>('form');
+  const [currentIndexSpec, setCurrentIndexSpec] = useState<IndexSpec>(indexSpec || {});
+  const [jsonError, setJsonError] = useState<Error | undefined>();
+
+  const issueWithCurrentIndexSpec = AutoForm.issueWithModel(currentIndexSpec, INDEX_SPEC_FIELDS);
+
+  return (
+    <Dialog
+      className="index-spec-dialog"
+      isOpen
+      onClose={onClose}
+      canOutsideClickClose={false}
+      title={title ?? 'Index spec'}
+    >
+      <FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
+      <div className="content">
+        {currentTab === 'form' ? (
+          <AutoForm
+            fields={INDEX_SPEC_FIELDS}
+            model={currentIndexSpec}
+            onChange={m => setCurrentIndexSpec(m)}
+          />
+        ) : (
+          <JsonInput
+            value={currentIndexSpec}
+            onChange={v => {
+              setCurrentIndexSpec(v);
+              setJsonError(undefined);
+            }}
+            onError={setJsonError}
+            issueWithValue={value => AutoForm.issueWithModel(value, INDEX_SPEC_FIELDS)}
+            height="100%"
+          />
+        )}
+      </div>
+      <div className={Classes.DIALOG_FOOTER}>
+        <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+          <Button text="Close" onClick={onClose} />
+          <Button
+            text="Save"
+            intent={Intent.PRIMARY}
+            disabled={Boolean(jsonError || issueWithCurrentIndexSpec)}
+            onClick={() => {
+              onSave(currentIndexSpec);
+              onClose();
+            }}
+          />
+        </div>
+      </div>
+    </Dialog>
+  );
+});

web-console/src/druid-models/index-spec/index-spec.tsx

@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Code } from '@blueprintjs/core';
+import React from 'react';
+
+import { Field } from '../../components';
+import { deepGet } from '../../utils';
+
+export interface IndexSpec {
+  bitmap?: Bitmap;
+  dimensionCompression?: string;
+  stringDictionaryEncoding?: { type: 'utf8' | 'frontCoded'; bucketSize: number };
+  metricCompression?: string;
+  longEncoding?: string;
+  jsonCompression?: string;
+}
+
+export interface Bitmap {
+  type: string;
+  compressRunOnSerialization?: boolean;
+}
+
+export function summarizeIndexSpec(indexSpec: IndexSpec | undefined): string {
+  if (!indexSpec) return '';
+
+  const { stringDictionaryEncoding, bitmap, longEncoding } = indexSpec;
+
+  const ret: string[] = [];
+  if (stringDictionaryEncoding) {
+    switch (stringDictionaryEncoding.type) {
+      case 'frontCoded':
+        ret.push(`frontCoded(${stringDictionaryEncoding.bucketSize || 4})`);
+        break;
+
+      default:
+        ret.push(stringDictionaryEncoding.type);
+        break;
+    }
+  }
+
+  if (bitmap) {
+    ret.push(bitmap.type);
+  }
+
+  if (longEncoding) {
+    ret.push(longEncoding);
+  }
+
+  return ret.join('; ');
+}
+
+export const INDEX_SPEC_FIELDS: Field<IndexSpec>[] = [
+  {
+    name: 'stringDictionaryEncoding.type',
+    label: 'String dictionary encoding',
+    type: 'string',
+    defaultValue: 'utf8',
+    suggestions: ['utf8', 'frontCoded'],
+    info: (
+      <>
+        Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt;
+        columns.
+      </>
+    ),
+  },
+  {
+    name: 'stringDictionaryEncoding.bucketSize',
+    label: 'String dictionary encoding bucket size',
+    type: 'number',
+    defaultValue: 4,
+    min: 1,
+    max: 128,
+    defined: spec => deepGet(spec, 'stringDictionaryEncoding.type') === 'frontCoded',
+    info: (
+      <>
+        The number of values to place in a bucket to perform delta encoding. Must be a power of 2,
+        maximum is 128.
+      </>
+    ),
+  },
+
+  {
+    name: 'bitmap.type',
+    label: 'Bitmap type',
+    type: 'string',
+    defaultValue: 'roaring',
+    suggestions: ['roaring', 'concise'],
+    info: <>Compression format for bitmap indexes.</>,
+  },
+  {
+    name: 'bitmap.compressRunOnSerialization',
+    label: 'Bitmap compress run on serialization',
+    type: 'boolean',
+    defaultValue: true,
+    defined: spec => (deepGet(spec, 'bitmap.type') || 'roaring') === 'roaring',
+    info: (
+      <>
+        Controls whether or not run-length encoding will be used when it is determined to be more
+        space-efficient.
+      </>
+    ),
+  },
+
+  {
+    name: 'dimensionCompression',
+    type: 'string',
+    defaultValue: 'lz4',
+    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
+    info: <>Compression format for dimension columns.</>,
+  },
+
+  {
+    name: 'longEncoding',
+    type: 'string',
+    defaultValue: 'longs',
+    suggestions: ['longs', 'auto'],
+    info: (
+      <>
+        Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
+        metrics. <Code>auto</Code> encodes the values using offset or lookup table depending on
+        column cardinality, and store them with variable size. <Code>longs</Code> stores the value
+        as-is with 8 bytes each.
+      </>
+    ),
+  },
+  {
+    name: 'metricCompression',
+    type: 'string',
+    defaultValue: 'lz4',
+    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
+    info: <>Compression format for primitive type metric columns.</>,
+  },
+
+  {
+    name: 'jsonCompression',
+    label: 'JSON compression',
+    type: 'string',
+    defaultValue: 'lz4',
+    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
+    info: <>Compression format to use for nested column raw data. </>,
+  },
+];

web-console/src/druid-models/index.ts

@@ -25,6 +25,7 @@ export * from './execution/execution';
 export * from './external-config/external-config';
 export * from './filter/filter';
 export * from './flatten-spec/flatten-spec';
+export * from './index-spec/index-spec';
 export * from './ingest-query-pattern/ingest-query-pattern';
 export * from './ingestion-spec/ingestion-spec';
 export * from './input-format/input-format';

web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx

@@ -21,6 +21,7 @@ import { range } from 'd3-array';
 import React from 'react';
 
 import { AutoForm, ExternalLink, Field } from '../../components';
+import { IndexSpecDialog } from '../../dialogs/index-spec-dialog/index-spec-dialog';
 import { getLink } from '../../links';
 import {
   allowKeys,
@@ -44,6 +45,7 @@ import {
   getDimensionSpecs,
   getDimensionSpecType,
 } from '../dimension-spec/dimension-spec';
+import { IndexSpec, summarizeIndexSpec } from '../index-spec/index-spec';
 import { InputFormat, issueWithInputFormat } from '../input-format/input-format';
 import {
   FILTER_SUGGESTIONS,
@@ -1379,6 +1381,7 @@ export interface TuningConfig {
   partitionsSpec?: PartitionsSpec;
   maxPendingPersists?: number;
   indexSpec?: IndexSpec;
+  indexSpecForIntermediatePersists?: IndexSpec;
   forceExtendableShardSpecs?: boolean;
   forceGuaranteedRollup?: boolean;
   reportParseExceptions?: boolean;
@@ -1869,103 +1872,38 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
   },
 
   {
-    name: 'spec.tuningConfig.indexSpec.bitmap.type',
+    name: 'spec.tuningConfig.indexSpec',
-    label: 'Index bitmap type',
+    type: 'custom',
-    type: 'string',
-    defaultValue: 'roaring',
-    suggestions: ['concise', 'roaring'],
     hideInMore: true,
-    info: <>Compression format for bitmap indexes.</>,
+    info: <>Defines segment storage format options to use at indexing time.</>,
+    placeholder: 'Default index spec',
+    customSummary: summarizeIndexSpec,
+    customDialog: ({ value, onValueChange, onClose }) => (
+      <IndexSpecDialog onClose={onClose} onSave={onValueChange} indexSpec={value} />
+    ),
   },
   {
-    name: 'spec.tuningConfig.indexSpec.bitmap.compressRunOnSerialization',
+    name: 'spec.tuningConfig.indexSpecForIntermediatePersists',
-    type: 'boolean',
+    type: 'custom',
-    defaultValue: true,
+    hideInMore: true,
-    defined: spec => deepGet(spec, 'spec.tuningConfig.indexSpec.bitmap.type') === 'roaring',
     info: (
       <>
-        Controls whether or not run-length encoding will be used when it is determined to be more
+        Defines segment storage format options to use at indexing time for intermediate persisted
-        space-efficient.
+        temporary segments.
       </>
     ),
+    placeholder: 'Default index spec',
+    customSummary: summarizeIndexSpec,
+    customDialog: ({ value, onValueChange, onClose }) => (
+      <IndexSpecDialog
+        title="Index spec for intermediate persists"
+        onClose={onClose}
+        onSave={onValueChange}
+        indexSpec={value}
+      />
+    ),
   },
 
-  {
-    name: 'spec.tuningConfig.indexSpec.dimensionCompression',
-    label: 'Index dimension compression',
-    type: 'string',
-    defaultValue: 'lz4',
-    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
-    hideInMore: true,
-    info: <>Compression format for dimension columns.</>,
-  },
-
-  {
-    name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type',
-    label: 'Index string dictionary encoding',
-    type: 'string',
-    defaultValue: 'utf8',
-    suggestions: ['utf8', 'frontCoded'],
-    hideInMore: true,
-    info: (
-      <>
-        Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt;
-        columns.
-      </>
-    ),
-  },
-  {
-    name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.bucketSize',
-    label: 'Index string dictionary encoding bucket size',
-    type: 'number',
-    defaultValue: 4,
-    min: 1,
-    max: 128,
-    defined: spec =>
-      deepGet(spec, 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type') === 'frontCoded',
-    hideInMore: true,
-    info: (
-      <>
-        The number of values to place in a bucket to perform delta encoding. Must be a power of 2,
-        maximum is 128.
-      </>
-    ),
-  },
-
-  {
-    name: 'spec.tuningConfig.indexSpec.metricCompression',
-    label: 'Index metric compression',
-    type: 'string',
-    defaultValue: 'lz4',
-    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
-    hideInMore: true,
-    info: <>Compression format for primitive type metric columns.</>,
-  },
-  {
-    name: 'spec.tuningConfig.indexSpec.longEncoding',
-    label: 'Index long encoding',
-    type: 'string',
-    defaultValue: 'longs',
-    suggestions: ['longs', 'auto'],
-    hideInMore: true,
-    info: (
-      <>
-        Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
-        metrics. <Code>auto</Code> encodes the values using offset or lookup table depending on
-        column cardinality, and store them with variable size. <Code>longs</Code> stores the value
-        as-is with 8 bytes each.
-      </>
-    ),
-  },
-  {
-    name: 'spec.tuningConfig.indexSpec.jsonCompression',
-    label: 'Index JSON compression',
-    type: 'string',
-    defaultValue: 'lz4',
-    suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
-    hideInMore: true,
-    info: <>Compression format to use for nested column raw data. </>,
-  },
   {
     name: 'spec.tuningConfig.splitHintSpec.maxSplitSize',
     type: 'number',
@@ -2172,18 +2110,6 @@ export function getTuningFormFields() {
   return TUNING_FORM_FIELDS;
 }
 
-export interface IndexSpec {
-  bitmap?: Bitmap;
-  dimensionCompression?: string;
-  metricCompression?: string;
-  longEncoding?: string;
-}
-
-export interface Bitmap {
-  type: string;
-  compressRunOnSerialization?: boolean;
-}
-
 // --------------
 
 export function updateIngestionType(

web-console/src/druid-models/workbench-query/workbench-query-part.ts

@@ -62,10 +62,10 @@ export class WorkbenchQueryPart {
   static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined {
     // Assuming the queryFragment is no parsable find the prefix that look like:
     // REPLACE<space>INTO<space><whatever><space>SELECT<space or EOF>
-    const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/)?.index;
+    const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index;
     if (typeof matchInsertReplaceIndex !== 'number') return;
 
-    const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/);
+    const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/i);
     const fragmentQuery = SqlQuery.maybeParse(
       queryFragment.substring(matchInsertReplaceIndex, matchEnd?.index) + ' SELECT * FROM t',
     );

web-console/src/druid-models/workbench-query/workbench-query.spec.ts

@@ -465,7 +465,7 @@ describe('WorkbenchQuery', () => {
     it('works with INSERT (unparsable)', () => {
       const sql = sane`
         -- Some comment
-        INSERT INTO trips2
+        INSERT into trips2
         SELECT
           TIME_PARSE(pickup_datetime) AS __time,
           *

web-console/src/helpers/spec-conversion.spec.ts

@@ -106,6 +106,9 @@ describe('spec conversion', () => {
             partitionDimension: 'isRobot',
             targetRowsPerSegment: 150000,
           },
+          indexSpec: {
+            dimensionCompression: 'lzf',
+          },
           forceGuaranteedRollup: true,
           maxNumConcurrentSubTasks: 4,
           maxParseExceptions: 3,
@@ -159,6 +162,9 @@ describe('spec conversion', () => {
       maxParseExceptions: 3,
       finalizeAggregations: false,
       maxNumTasks: 5,
+      indexSpec: {
+        dimensionCompression: 'lzf',
+      },
     });
   });
 

web-console/src/helpers/spec-conversion.ts

@@ -70,6 +70,11 @@ export function convertSpecToSql(spec: any): QueryWithContext {
     groupByEnableMultiValueUnnesting: false,
   };
 
+  const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
+  if (indexSpec) {
+    context.indexSpec = indexSpec;
+  }
+
   const lines: string[] = [];
 
   const rollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup') ?? true;

web-console/src/views/workbench-view/run-panel/run-panel.tsx

@@ -33,6 +33,7 @@ import React, { useCallback, useMemo, useState } from 'react';
 
 import { MenuCheckbox, MenuTristate } from '../../../components';
 import { EditContextDialog, StringInputDialog } from '../../../dialogs';
+import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
 import {
   changeDurableShuffleStorage,
   changeFinalizeAggregations,
@@ -51,9 +52,12 @@ import {
   getUseApproximateCountDistinct,
   getUseApproximateTopN,
   getUseCache,
+  IndexSpec,
+  QueryContext,
+  summarizeIndexSpec,
   WorkbenchQuery,
 } from '../../../druid-models';
-import { pluralIfNeeded, tickIcon } from '../../../utils';
+import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils';
 import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
 
 import './run-panel.scss';
@@ -94,6 +98,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines } = props;
   const [editContextDialogOpen, setEditContextDialogOpen] = useState(false);
   const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false);
+  const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState<IndexSpec | undefined>();
 
   const emptyQuery = query.isEmptyQuery();
   const ingestMode = query.isIngestQuery();
@@ -104,6 +109,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   const finalizeAggregations = getFinalizeAggregations(queryContext);
   const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
   const durableShuffleStorage = getDurableShuffleStorage(queryContext);
+  const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
   const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext);
   const useApproximateTopN = getUseApproximateTopN(queryContext);
   const useCache = getUseCache(queryContext);
@@ -157,6 +163,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
     );
   }
 
+  function changeQueryContext(queryContext: QueryContext) {
+    onQueryChange(query.changeQueryContext(queryContext));
+  }
+
   const availableEngines = ([undefined] as (DruidEngine | undefined)[]).concat(queryEngines);
 
   function offsetOptions(): JSX.Element[] {
@@ -170,9 +180,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
           icon={tickIcon(offset === timezone)}
           text={offset}
           shouldDismissPopover={false}
-          onClick={() => {
+          onClick={() => changeQueryContext(changeTimezone(queryContext, offset))}
-            onQueryChange(query.changeQueryContext(changeTimezone(queryContext, offset)));
-          }}
         />,
       );
     }
@@ -233,11 +241,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       icon={tickIcon(!timezone)}
                       text="Default"
                       shouldDismissPopover={false}
-                      onClick={() => {
+                      onClick={() => changeQueryContext(changeTimezone(queryContext, undefined))}
-                        onQueryChange(
-                          query.changeQueryContext(changeTimezone(queryContext, undefined)),
-                        );
-                      }}
                     />
                     <MenuItem icon={tickIcon(String(timezone).includes('/'))} text="Named">
                       {NAMED_TIMEZONES.map(namedTimezone => (
@@ -246,11 +250,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                           icon={tickIcon(namedTimezone === timezone)}
                           text={namedTimezone}
                           shouldDismissPopover={false}
-                          onClick={() => {
+                          onClick={() =>
-                            onQueryChange(
+                            changeQueryContext(changeTimezone(queryContext, namedTimezone))
-                              query.changeQueryContext(changeTimezone(queryContext, namedTimezone)),
+                          }
-                            );
-                          }}
                         />
                       ))}
                     </MenuItem>
@@ -276,11 +278,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                           key={String(v)}
                           icon={tickIcon(v === maxParseExceptions)}
                           text={v === -1 ? '∞ (-1)' : String(v)}
-                          onClick={() => {
+                          onClick={() =>
-                            onQueryChange(
+                            changeQueryContext(changeMaxParseExceptions(queryContext, v))
-                              query.changeQueryContext(changeMaxParseExceptions(queryContext, v)),
+                          }
-                            );
-                          }}
                           shouldDismissPopover={false}
                         />
                       ))}
@@ -290,35 +290,36 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                       text="Finalize aggregations"
                       value={finalizeAggregations}
                       undefinedEffectiveValue={!ingestMode}
-                      onValueChange={v => {
+                      onValueChange={v =>
-                        onQueryChange(
+                        changeQueryContext(changeFinalizeAggregations(queryContext, v))
-                          query.changeQueryContext(changeFinalizeAggregations(queryContext, v)),
+                      }
-                        );
-                      }}
                     />
                     <MenuTristate
                       icon={IconNames.FORK}
                       text="Enable GroupBy multi-value unnesting"
                       value={groupByEnableMultiValueUnnesting}
                       undefinedEffectiveValue={!ingestMode}
-                      onValueChange={v => {
+                      onValueChange={v =>
-                        onQueryChange(
+                        changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v))
-                          query.changeQueryContext(
+                      }
-                            changeGroupByEnableMultiValueUnnesting(queryContext, v),
+                    />
-                          ),
+                    <MenuItem
-                        );
+                      icon={IconNames.TH_DERIVED}
+                      text="Edit index spec"
+                      label={summarizeIndexSpec(indexSpec)}
+                      shouldDismissPopover={false}
+                      onClick={() => {
+                        setIndexSpecDialogSpec(indexSpec || {});
                       }}
                     />
                     <MenuCheckbox
                       checked={durableShuffleStorage}
                       text="Durable shuffle storage"
-                      onChange={() => {
+                      onChange={() =>
-                        onQueryChange(
+                        changeQueryContext(
-                          query.changeQueryContext(
                           changeDurableShuffleStorage(queryContext, !durableShuffleStorage),
-                          ),
+                        )
-                        );
+                      }
-                      }}
                     />
                   </>
                 ) : (
@@ -326,22 +327,16 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                     <MenuCheckbox
                       checked={useCache}
                       text="Use cache"
-                      onChange={() => {
+                      onChange={() => changeQueryContext(changeUseCache(queryContext, !useCache))}
-                        onQueryChange(
-                          query.changeQueryContext(changeUseCache(queryContext, !useCache)),
-                        );
-                      }}
                     />
                     <MenuCheckbox
                       checked={useApproximateTopN}
                       text="Use approximate TopN"
-                      onChange={() => {
+                      onChange={() =>
-                        onQueryChange(
+                        changeQueryContext(
-                          query.changeQueryContext(
                           changeUseApproximateTopN(queryContext, !useApproximateTopN),
-                          ),
+                        )
-                        );
+                      }
-                      }}
                     />
                   </>
                 )}
@@ -349,16 +344,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
                   <MenuCheckbox
                     checked={useApproximateCountDistinct}
                     text="Use approximate COUNT(DISTINCT)"
-                    onChange={() => {
+                    onChange={() =>
-                      onQueryChange(
+                      changeQueryContext(
-                        query.changeQueryContext(
                         changeUseApproximateCountDistinct(
                           queryContext,
                           !useApproximateCountDistinct,
                         ),
-                        ),
+                      )
-                      );
+                    }
-                    }}
                   />
                 )}
                 <MenuCheckbox
@@ -382,12 +375,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
             />
           </Popover2>
           {effectiveEngine === 'sql-msq-task' && (
-            <MaxTasksButton
+            <MaxTasksButton queryContext={queryContext} changeQueryContext={changeQueryContext} />
-              queryContext={queryContext}
-              changeQueryContext={queryContext =>
-                onQueryChange(query.changeQueryContext(queryContext))
-              }
-            />
           )}
         </ButtonGroup>
       )}
@@ -399,10 +387,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
       {editContextDialogOpen && (
         <EditContextDialog
           queryContext={queryContext}
-          onQueryContextChange={newContext => {
+          onQueryContextChange={changeQueryContext}
-            if (!onQueryChange) return;
-            onQueryChange(query.changeQueryContext(newContext));
-          }}
           onClose={() => {
             setEditContextDialogOpen(false);
           }}
@@ -413,10 +398,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
           title="Custom timezone"
           placeholder="Etc/UTC"
           maxLength={50}
-          onSubmit={tz => onQueryChange(query.changeQueryContext(changeTimezone(queryContext, tz)))}
+          onSubmit={tz => changeQueryContext(changeTimezone(queryContext, tz))}
           onClose={() => setCustomTimezoneDialogOpen(false)}
         />
       )}
+      {indexSpecDialogSpec && (
+        <IndexSpecDialog
+          onClose={() => setIndexSpecDialogSpec(undefined)}
+          onSave={indexSpec => changeQueryContext({ ...queryContext, indexSpec })}
+          indexSpec={indexSpecDialogSpec}
+        />
+      )}
     </div>
   );
 });