Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: Index spec dialog (#13425) 

* add index spec dialog

* add sanpshot
This commit is contained in:
Vadim Ogievetsky 2022-11-28 11:40:45 -08:00 committed by GitHub
parent b12e5f300e
commit a2d5e335f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 801 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
web-console/src/components/auto-form/auto-form.scss
View File

@ -28,4 +28,8 @@
right: 0;
}
}
.custom-input input {
cursor: pointer;
}
}

55
web-console/src/components/auto-form/auto-form.tsx
View File

@ -16,7 +16,14 @@
* limitations under the License.
*/
import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core';
import {
Button,
ButtonGroup,
FormGroup,
InputGroup,
Intent,
NumericInput,
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
@ -46,7 +53,8 @@ export interface Field<M> {
| 'boolean'
| 'string-array'
| 'json'
| 'interval';
| 'interval'
| 'custom';
defaultValue?: any;
emptyValue?: any;
suggestions?: Functor<M, Suggestion[]>;
@ -64,6 +72,13 @@ export interface Field<M> {
valueAdjustment?: (value: any) => any;
adjustment?: (model: Partial<M>) => Partial<M>;
issueWithValue?: (value: any) => string | undefined;
customSummary?: (v: any) => string;
customDialog?: (o: {
value: any;
onValueChange: (v: any) => void;
onClose: () => void;
}) => JSX.Element;
}
interface ComputedFieldValues {
@ -84,6 +99,7 @@ export interface AutoFormProps<M> {
export interface AutoFormState {
showMore: boolean;
customDialog?: JSX.Element;
}
export class AutoForm<T extends Record<string, any>> extends React.PureComponent<
@ -395,6 +411,36 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
);
}
private renderCustomInput(field: Field<T>): JSX.Element {
const { model } = this.props;
const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field);
const effectiveValue = modelValue || defaultValue;
const onEdit = () => {
this.setState({
customDialog: field.customDialog?.({
value: effectiveValue,
onValueChange: v => this.fieldChange(field, v),
onClose: () => {
this.setState({ customDialog: undefined });
},
}),
});
};
return (
<InputGroup
className="custom-input"
value={(field.customSummary || String)(effectiveValue)}
intent={required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined}
readOnly
placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')}
rightElement={<Button icon={IconNames.EDIT} minimal onClick={onEdit} />}
onClick={onEdit}
/>
);
}
renderFieldInput(field: Field<T>) {
switch (field.type) {
case 'number':
@ -413,6 +459,8 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
return this.renderJsonInput(field);
case 'interval':
return this.renderIntervalInput(field);
case 'custom':
return this.renderCustomInput(field);
default:
throw new Error(`unknown field type '${field.type}'`);
}
@ -464,7 +512,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
render(): JSX.Element {
const { fields, model, showCustom } = this.props;
const { showMore } = this.state;
const { showMore, customDialog } = this.state;
let shouldShowMore = false;
const shownFields = fields.filter(field => {
@ -489,6 +537,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
{model && shownFields.map(this.renderField)}
{model && showCustom && showCustom(model) && this.renderCustom()}
{shouldShowMore && this.renderMoreOrLess()}
{customDialog}
</div>
);
}

317
web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap Normal file
View File

@ -0,0 +1,317 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`IndexSpecDialog matches snapshot with indexSpec 1`] = `
<Blueprint4.Dialog
canOutsideClickClose={false}
className="index-spec-dialog"
isOpen={true}
onClose={[Function]}
title="Index spec"
>
<Memo(FormJsonSelector)
onChange={[Function]}
tab="form"
/>
<div
className="content"
>
<AutoForm
fields={
Array [
Object {
"defaultValue": "utf8",
"info": <React.Fragment>
Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns.
</React.Fragment>,
"label": "String dictionary encoding",
"name": "stringDictionaryEncoding.type",
"suggestions": Array [
"utf8",
"frontCoded",
],
"type": "string",
},
Object {
"defaultValue": 4,
"defined": [Function],
"info": <React.Fragment>
The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128.
</React.Fragment>,
"label": "String dictionary encoding bucket size",
"max": 128,
"min": 1,
"name": "stringDictionaryEncoding.bucketSize",
"type": "number",
},
Object {
"defaultValue": "roaring",
"info": <React.Fragment>
Compression format for bitmap indexes.
</React.Fragment>,
"label": "Bitmap type",
"name": "bitmap.type",
"suggestions": Array [
"roaring",
"concise",
],
"type": "string",
},
Object {
"defaultValue": true,
"defined": [Function],
"info": <React.Fragment>
Controls whether or not run-length encoding will be used when it is determined to be more space-efficient.
</React.Fragment>,
"label": "Bitmap compress run on serialization",
"name": "bitmap.compressRunOnSerialization",
"type": "boolean",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format for dimension columns.
</React.Fragment>,
"name": "dimensionCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
Object {
"defaultValue": "longs",
"info": <React.Fragment>
Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics.
<Unknown>
auto
</Unknown>
encodes the values using offset or lookup table depending on column cardinality, and store them with variable size.
<Unknown>
longs
</Unknown>
stores the value as-is with 8 bytes each.
</React.Fragment>,
"name": "longEncoding",
"suggestions": Array [
"longs",
"auto",
],
"type": "string",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format for primitive type metric columns.
</React.Fragment>,
"name": "metricCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format to use for nested column raw data.
</React.Fragment>,
"label": "JSON compression",
"name": "jsonCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
]
}
model={
Object {
"dimensionCompression": "lzf",
}
}
onChange={[Function]}
/>
</div>
<div
className="bp4-dialog-footer"
>
<div
className="bp4-dialog-footer-actions"
>
<Blueprint4.Button
onClick={[Function]}
text="Close"
/>
<Blueprint4.Button
disabled={false}
intent="primary"
onClick={[Function]}
text="Save"
/>
</div>
</div>
</Blueprint4.Dialog>
`;
exports[`IndexSpecDialog matches snapshot without compactionConfig 1`] = `
<Blueprint4.Dialog
canOutsideClickClose={false}
className="index-spec-dialog"
isOpen={true}
onClose={[Function]}
title="Index spec"
>
<Memo(FormJsonSelector)
onChange={[Function]}
tab="form"
/>
<div
className="content"
>
<AutoForm
fields={
Array [
Object {
"defaultValue": "utf8",
"info": <React.Fragment>
Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns.
</React.Fragment>,
"label": "String dictionary encoding",
"name": "stringDictionaryEncoding.type",
"suggestions": Array [
"utf8",
"frontCoded",
],
"type": "string",
},
Object {
"defaultValue": 4,
"defined": [Function],
"info": <React.Fragment>
The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128.
</React.Fragment>,
"label": "String dictionary encoding bucket size",
"max": 128,
"min": 1,
"name": "stringDictionaryEncoding.bucketSize",
"type": "number",
},
Object {
"defaultValue": "roaring",
"info": <React.Fragment>
Compression format for bitmap indexes.
</React.Fragment>,
"label": "Bitmap type",
"name": "bitmap.type",
"suggestions": Array [
"roaring",
"concise",
],
"type": "string",
},
Object {
"defaultValue": true,
"defined": [Function],
"info": <React.Fragment>
Controls whether or not run-length encoding will be used when it is determined to be more space-efficient.
</React.Fragment>,
"label": "Bitmap compress run on serialization",
"name": "bitmap.compressRunOnSerialization",
"type": "boolean",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format for dimension columns.
</React.Fragment>,
"name": "dimensionCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
Object {
"defaultValue": "longs",
"info": <React.Fragment>
Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics.
<Unknown>
auto
</Unknown>
encodes the values using offset or lookup table depending on column cardinality, and store them with variable size.
<Unknown>
longs
</Unknown>
stores the value as-is with 8 bytes each.
</React.Fragment>,
"name": "longEncoding",
"suggestions": Array [
"longs",
"auto",
],
"type": "string",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format for primitive type metric columns.
</React.Fragment>,
"name": "metricCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
Object {
"defaultValue": "lz4",
"info": <React.Fragment>
Compression format to use for nested column raw data.
</React.Fragment>,
"label": "JSON compression",
"name": "jsonCompression",
"suggestions": Array [
"lz4",
"lzf",
"zstd",
"uncompressed",
],
"type": "string",
},
]
}
model={Object {}}
onChange={[Function]}
/>
</div>
<div
className="bp4-dialog-footer"
>
<div
className="bp4-dialog-footer-actions"
>
<Blueprint4.Button
onClick={[Function]}
text="Close"
/>
<Blueprint4.Button
disabled={false}
intent="primary"
onClick={[Function]}
text="Save"
/>
</div>
</div>
</Blueprint4.Dialog>
`;

36
web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../variables';
.index-spec-dialog {
&.#{$bp-ns}-dialog {
height: 70vh;
}
.form-json-selector {
margin: 15px;
}
.content {
margin: 0 15px 10px 0;
padding: 0 5px 0 15px;
flex: 1;
overflow: auto;
}
}

44
web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { shallow } from 'enzyme';
import React from 'react';
import { IndexSpecDialog } from './index-spec-dialog';
describe('IndexSpecDialog', () => {
it('matches snapshot without compactionConfig', () => {
const compactionDialog = shallow(
<IndexSpecDialog onClose={() => {}} onSave={() => {}} indexSpec={undefined} />,
);
expect(compactionDialog).toMatchSnapshot();
});
it('matches snapshot with indexSpec', () => {
const compactionDialog = shallow(
<IndexSpecDialog
onClose={() => {}}
onSave={() => {}}
indexSpec={{
dimensionCompression: 'lzf',
}}
/>,
);
expect(compactionDialog).toMatchSnapshot();
});
});

88
web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
import React, { useState } from 'react';
import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components';
import { INDEX_SPEC_FIELDS, IndexSpec } from '../../druid-models';
import './index-spec-dialog.scss';
export interface IndexSpecDialogProps {
title?: string;
onClose: () => void;
onSave: (indexSpec: IndexSpec) => void;
indexSpec: IndexSpec | undefined;
}
export const IndexSpecDialog = React.memo(function IndexSpecDialog(props: IndexSpecDialogProps) {
const { title, indexSpec, onSave, onClose } = props;
const [currentTab, setCurrentTab] = useState<FormJsonTabs>('form');
const [currentIndexSpec, setCurrentIndexSpec] = useState<IndexSpec>(indexSpec || {});
const [jsonError, setJsonError] = useState<Error | undefined>();
const issueWithCurrentIndexSpec = AutoForm.issueWithModel(currentIndexSpec, INDEX_SPEC_FIELDS);
return (
<Dialog
className="index-spec-dialog"
isOpen
onClose={onClose}
canOutsideClickClose={false}
title={title ?? 'Index spec'}
>
<FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
<div className="content">
{currentTab === 'form' ? (
<AutoForm
fields={INDEX_SPEC_FIELDS}
model={currentIndexSpec}
onChange={m => setCurrentIndexSpec(m)}
/>
) : (
<JsonInput
value={currentIndexSpec}
onChange={v => {
setCurrentIndexSpec(v);
setJsonError(undefined);
}}
onError={setJsonError}
issueWithValue={value => AutoForm.issueWithModel(value, INDEX_SPEC_FIELDS)}
height="100%"
/>
)}
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Button text="Close" onClick={onClose} />
<Button
text="Save"
intent={Intent.PRIMARY}
disabled={Boolean(jsonError || issueWithCurrentIndexSpec)}
onClick={() => {
onSave(currentIndexSpec);
onClose();
}}
/>
</div>
</div>
</Dialog>
);
});

158
web-console/src/druid-models/index-spec/index-spec.tsx Normal file
View File

@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Code } from '@blueprintjs/core';
import React from 'react';
import { Field } from '../../components';
import { deepGet } from '../../utils';
export interface IndexSpec {
bitmap?: Bitmap;
dimensionCompression?: string;
stringDictionaryEncoding?: { type: 'utf8' | 'frontCoded'; bucketSize: number };
metricCompression?: string;
longEncoding?: string;
jsonCompression?: string;
}
export interface Bitmap {
type: string;
compressRunOnSerialization?: boolean;
}
export function summarizeIndexSpec(indexSpec: IndexSpec | undefined): string {
if (!indexSpec) return '';
const { stringDictionaryEncoding, bitmap, longEncoding } = indexSpec;
const ret: string[] = [];
if (stringDictionaryEncoding) {
switch (stringDictionaryEncoding.type) {
case 'frontCoded':
ret.push(`frontCoded(${stringDictionaryEncoding.bucketSize || 4})`);
break;
default:
ret.push(stringDictionaryEncoding.type);
break;
}
}
if (bitmap) {
ret.push(bitmap.type);
}
if (longEncoding) {
ret.push(longEncoding);
}
return ret.join('; ');
}
export const INDEX_SPEC_FIELDS: Field<IndexSpec>[] = [
{
name: 'stringDictionaryEncoding.type',
label: 'String dictionary encoding',
type: 'string',
defaultValue: 'utf8',
suggestions: ['utf8', 'frontCoded'],
info: (
<>
Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt;
columns.
</>
),
},
{
name: 'stringDictionaryEncoding.bucketSize',
label: 'String dictionary encoding bucket size',
type: 'number',
defaultValue: 4,
min: 1,
max: 128,
defined: spec => deepGet(spec, 'stringDictionaryEncoding.type') === 'frontCoded',
info: (
<>
The number of values to place in a bucket to perform delta encoding. Must be a power of 2,
maximum is 128.
</>
),
},
{
name: 'bitmap.type',
label: 'Bitmap type',
type: 'string',
defaultValue: 'roaring',
suggestions: ['roaring', 'concise'],
info: <>Compression format for bitmap indexes.</>,
},
{
name: 'bitmap.compressRunOnSerialization',
label: 'Bitmap compress run on serialization',
type: 'boolean',
defaultValue: true,
defined: spec => (deepGet(spec, 'bitmap.type') || 'roaring') === 'roaring',
info: (
<>
Controls whether or not run-length encoding will be used when it is determined to be more
space-efficient.
</>
),
},
{
name: 'dimensionCompression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
info: <>Compression format for dimension columns.</>,
},
{
name: 'longEncoding',
type: 'string',
defaultValue: 'longs',
suggestions: ['longs', 'auto'],
info: (
<>
Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
metrics. <Code>auto</Code> encodes the values using offset or lookup table depending on
column cardinality, and store them with variable size. <Code>longs</Code> stores the value
as-is with 8 bytes each.
</>
),
},
{
name: 'metricCompression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
info: <>Compression format for primitive type metric columns.</>,
},
{
name: 'jsonCompression',
label: 'JSON compression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
info: <>Compression format to use for nested column raw data. </>,
},
];

1
web-console/src/druid-models/index.ts
View File

@ -25,6 +25,7 @@ export * from './execution/execution';
export * from './external-config/external-config';
export * from './filter/filter';
export * from './flatten-spec/flatten-spec';
export * from './index-spec/index-spec';
export * from './ingest-query-pattern/ingest-query-pattern';
export * from './ingestion-spec/ingestion-spec';
export * from './input-format/input-format';

126
web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
View File

@ -21,6 +21,7 @@ import { range } from 'd3-array';
import React from 'react';
import { AutoForm, ExternalLink, Field } from '../../components';
import { IndexSpecDialog } from '../../dialogs/index-spec-dialog/index-spec-dialog';
import { getLink } from '../../links';
import {
allowKeys,
@ -44,6 +45,7 @@ import {
getDimensionSpecs,
getDimensionSpecType,
} from '../dimension-spec/dimension-spec';
import { IndexSpec, summarizeIndexSpec } from '../index-spec/index-spec';
import { InputFormat, issueWithInputFormat } from '../input-format/input-format';
import {
FILTER_SUGGESTIONS,
@ -1379,6 +1381,7 @@ export interface TuningConfig {
partitionsSpec?: PartitionsSpec;
maxPendingPersists?: number;
indexSpec?: IndexSpec;
indexSpecForIntermediatePersists?: IndexSpec;
forceExtendableShardSpecs?: boolean;
forceGuaranteedRollup?: boolean;
reportParseExceptions?: boolean;
@ -1869,103 +1872,38 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
},
{
name: 'spec.tuningConfig.indexSpec.bitmap.type',
label: 'Index bitmap type',
type: 'string',
defaultValue: 'roaring',
suggestions: ['concise', 'roaring'],
name: 'spec.tuningConfig.indexSpec',
type: 'custom',
hideInMore: true,
info: <>Compression format for bitmap indexes.</>,
info: <>Defines segment storage format options to use at indexing time.</>,
placeholder: 'Default index spec',
customSummary: summarizeIndexSpec,
customDialog: ({ value, onValueChange, onClose }) => (
<IndexSpecDialog onClose={onClose} onSave={onValueChange} indexSpec={value} />
),
},
{
name: 'spec.tuningConfig.indexSpec.bitmap.compressRunOnSerialization',
type: 'boolean',
defaultValue: true,
defined: spec => deepGet(spec, 'spec.tuningConfig.indexSpec.bitmap.type') === 'roaring',
name: 'spec.tuningConfig.indexSpecForIntermediatePersists',
type: 'custom',
hideInMore: true,
info: (
<>
Controls whether or not run-length encoding will be used when it is determined to be more
space-efficient.
Defines segment storage format options to use at indexing time for intermediate persisted
temporary segments.
</>
),
placeholder: 'Default index spec',
customSummary: summarizeIndexSpec,
customDialog: ({ value, onValueChange, onClose }) => (
<IndexSpecDialog
title="Index spec for intermediate persists"
onClose={onClose}
onSave={onValueChange}
indexSpec={value}
/>
),
},
{
name: 'spec.tuningConfig.indexSpec.dimensionCompression',
label: 'Index dimension compression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
hideInMore: true,
info: <>Compression format for dimension columns.</>,
},
{
name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type',
label: 'Index string dictionary encoding',
type: 'string',
defaultValue: 'utf8',
suggestions: ['utf8', 'frontCoded'],
hideInMore: true,
info: (
<>
Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt;
columns.
</>
),
},
{
name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.bucketSize',
label: 'Index string dictionary encoding bucket size',
type: 'number',
defaultValue: 4,
min: 1,
max: 128,
defined: spec =>
deepGet(spec, 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type') === 'frontCoded',
hideInMore: true,
info: (
<>
The number of values to place in a bucket to perform delta encoding. Must be a power of 2,
maximum is 128.
</>
),
},
{
name: 'spec.tuningConfig.indexSpec.metricCompression',
label: 'Index metric compression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
hideInMore: true,
info: <>Compression format for primitive type metric columns.</>,
},
{
name: 'spec.tuningConfig.indexSpec.longEncoding',
label: 'Index long encoding',
type: 'string',
defaultValue: 'longs',
suggestions: ['longs', 'auto'],
hideInMore: true,
info: (
<>
Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
metrics. <Code>auto</Code> encodes the values using offset or lookup table depending on
column cardinality, and store them with variable size. <Code>longs</Code> stores the value
as-is with 8 bytes each.
</>
),
},
{
name: 'spec.tuningConfig.indexSpec.jsonCompression',
label: 'Index JSON compression',
type: 'string',
defaultValue: 'lz4',
suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'],
hideInMore: true,
info: <>Compression format to use for nested column raw data. </>,
},
{
name: 'spec.tuningConfig.splitHintSpec.maxSplitSize',
type: 'number',
@ -2172,18 +2110,6 @@ export function getTuningFormFields() {
return TUNING_FORM_FIELDS;
}
export interface IndexSpec {
bitmap?: Bitmap;
dimensionCompression?: string;
metricCompression?: string;
longEncoding?: string;
}
export interface Bitmap {
type: string;
compressRunOnSerialization?: boolean;
}
// --------------
export function updateIngestionType(

4
web-console/src/druid-models/workbench-query/workbench-query-part.ts
View File

@ -62,10 +62,10 @@ export class WorkbenchQueryPart {
static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined {
// Assuming the queryFragment is no parsable find the prefix that look like:
// REPLACE<space>INTO<space><whatever><space>SELECT<space or EOF>
const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/)?.index;
const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index;
if (typeof matchInsertReplaceIndex !== 'number') return;
const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/);
const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/i);
const fragmentQuery = SqlQuery.maybeParse(
queryFragment.substring(matchInsertReplaceIndex, matchEnd?.index) + ' SELECT * FROM t',
);

2
web-console/src/druid-models/workbench-query/workbench-query.spec.ts
View File

@ -465,7 +465,7 @@ describe('WorkbenchQuery', () => {
it('works with INSERT (unparsable)', () => {
const sql = sane`
-- Some comment
INSERT INTO trips2
INSERT into trips2
SELECT
TIME_PARSE(pickup_datetime) AS __time,
*

6
web-console/src/helpers/spec-conversion.spec.ts
View File

@ -106,6 +106,9 @@ describe('spec conversion', () => {
partitionDimension: 'isRobot',
targetRowsPerSegment: 150000,
},
indexSpec: {
dimensionCompression: 'lzf',
},
forceGuaranteedRollup: true,
maxNumConcurrentSubTasks: 4,
maxParseExceptions: 3,
@ -159,6 +162,9 @@ describe('spec conversion', () => {
maxParseExceptions: 3,
finalizeAggregations: false,
maxNumTasks: 5,
indexSpec: {
dimensionCompression: 'lzf',
},
});
});

5
web-console/src/helpers/spec-conversion.ts
View File

@ -70,6 +70,11 @@ export function convertSpecToSql(spec: any): QueryWithContext {
groupByEnableMultiValueUnnesting: false,
};
const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
if (indexSpec) {
context.indexSpec = indexSpec;
}
const lines: string[] = [];
const rollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup') ?? true;

130
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -33,6 +33,7 @@ import React, { useCallback, useMemo, useState } from 'react';
import { MenuCheckbox, MenuTristate } from '../../../components';
import { EditContextDialog, StringInputDialog } from '../../../dialogs';
import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
import {
changeDurableShuffleStorage,
changeFinalizeAggregations,
@ -51,9 +52,12 @@ import {
getUseApproximateCountDistinct,
getUseApproximateTopN,
getUseCache,
IndexSpec,
QueryContext,
summarizeIndexSpec,
WorkbenchQuery,
} from '../../../druid-models';
import { pluralIfNeeded, tickIcon } from '../../../utils';
import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils';
import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
import './run-panel.scss';
@ -94,6 +98,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines } = props;
const [editContextDialogOpen, setEditContextDialogOpen] = useState(false);
const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false);
const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState<IndexSpec | undefined>();
const emptyQuery = query.isEmptyQuery();
const ingestMode = query.isIngestQuery();
@ -104,6 +109,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const finalizeAggregations = getFinalizeAggregations(queryContext);
const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
const durableShuffleStorage = getDurableShuffleStorage(queryContext);
const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext);
const useApproximateTopN = getUseApproximateTopN(queryContext);
const useCache = getUseCache(queryContext);
@ -157,6 +163,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
);
}
function changeQueryContext(queryContext: QueryContext) {
onQueryChange(query.changeQueryContext(queryContext));
}
const availableEngines = ([undefined] as (DruidEngine | undefined)[]).concat(queryEngines);
function offsetOptions(): JSX.Element[] {
@ -170,9 +180,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
icon={tickIcon(offset === timezone)}
text={offset}
shouldDismissPopover={false}
onClick={() => {
onQueryChange(query.changeQueryContext(changeTimezone(queryContext, offset)));
}}
onClick={() => changeQueryContext(changeTimezone(queryContext, offset))}
/>,
);
}
@ -233,11 +241,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
icon={tickIcon(!timezone)}
text="Default"
shouldDismissPopover={false}
onClick={() => {
onQueryChange(
query.changeQueryContext(changeTimezone(queryContext, undefined)),
);
}}
onClick={() => changeQueryContext(changeTimezone(queryContext, undefined))}
/>
<MenuItem icon={tickIcon(String(timezone).includes('/'))} text="Named">
{NAMED_TIMEZONES.map(namedTimezone => (
@ -246,11 +250,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
icon={tickIcon(namedTimezone === timezone)}
text={namedTimezone}
shouldDismissPopover={false}
onClick={() => {
onQueryChange(
query.changeQueryContext(changeTimezone(queryContext, namedTimezone)),
);
}}
onClick={() =>
changeQueryContext(changeTimezone(queryContext, namedTimezone))
}
/>
))}
</MenuItem>
@ -276,11 +278,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
key={String(v)}
icon={tickIcon(v === maxParseExceptions)}
text={v === -1 ? '∞ (-1)' : String(v)}
onClick={() => {
onQueryChange(
query.changeQueryContext(changeMaxParseExceptions(queryContext, v)),
);
}}
onClick={() =>
changeQueryContext(changeMaxParseExceptions(queryContext, v))
}
shouldDismissPopover={false}
/>
))}
@ -290,35 +290,36 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
text="Finalize aggregations"
value={finalizeAggregations}
undefinedEffectiveValue={!ingestMode}
onValueChange={v => {
onQueryChange(
query.changeQueryContext(changeFinalizeAggregations(queryContext, v)),
);
}}
onValueChange={v =>
changeQueryContext(changeFinalizeAggregations(queryContext, v))
}
/>
<MenuTristate
icon={IconNames.FORK}
text="Enable GroupBy multi-value unnesting"
value={groupByEnableMultiValueUnnesting}
undefinedEffectiveValue={!ingestMode}
onValueChange={v => {
onQueryChange(
query.changeQueryContext(
changeGroupByEnableMultiValueUnnesting(queryContext, v),
),
);
onValueChange={v =>
changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v))
}
/>
<MenuItem
icon={IconNames.TH_DERIVED}
text="Edit index spec"
label={summarizeIndexSpec(indexSpec)}
shouldDismissPopover={false}
onClick={() => {
setIndexSpecDialogSpec(indexSpec || {});
}}
/>
<MenuCheckbox
checked={durableShuffleStorage}
text="Durable shuffle storage"
onChange={() => {
onQueryChange(
query.changeQueryContext(
changeDurableShuffleStorage(queryContext, !durableShuffleStorage),
),
);
}}
onChange={() =>
changeQueryContext(
changeDurableShuffleStorage(queryContext, !durableShuffleStorage),
)
}
/>
</>
) : (
@ -326,22 +327,16 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
<MenuCheckbox
checked={useCache}
text="Use cache"
onChange={() => {
onQueryChange(
query.changeQueryContext(changeUseCache(queryContext, !useCache)),
);
}}
onChange={() => changeQueryContext(changeUseCache(queryContext, !useCache))}
/>
<MenuCheckbox
checked={useApproximateTopN}
text="Use approximate TopN"
onChange={() => {
onQueryChange(
query.changeQueryContext(
changeUseApproximateTopN(queryContext, !useApproximateTopN),
),
);
}}
onChange={() =>
changeQueryContext(
changeUseApproximateTopN(queryContext, !useApproximateTopN),
)
}
/>
</>
)}
@ -349,16 +344,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
<MenuCheckbox
checked={useApproximateCountDistinct}
text="Use approximate COUNT(DISTINCT)"
onChange={() => {
onQueryChange(
query.changeQueryContext(
changeUseApproximateCountDistinct(
queryContext,
!useApproximateCountDistinct,
),
onChange={() =>
changeQueryContext(
changeUseApproximateCountDistinct(
queryContext,
!useApproximateCountDistinct,
),
);
}}
)
}
/>
)}
<MenuCheckbox
@ -382,12 +375,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
/>
</Popover2>
{effectiveEngine === 'sql-msq-task' && (
<MaxTasksButton
queryContext={queryContext}
changeQueryContext={queryContext =>
onQueryChange(query.changeQueryContext(queryContext))
}
/>
<MaxTasksButton queryContext={queryContext} changeQueryContext={changeQueryContext} />
)}
</ButtonGroup>
)}
@ -399,10 +387,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
{editContextDialogOpen && (
<EditContextDialog
queryContext={queryContext}
onQueryContextChange={newContext => {
if (!onQueryChange) return;
onQueryChange(query.changeQueryContext(newContext));
}}
onQueryContextChange={changeQueryContext}
onClose={() => {
setEditContextDialogOpen(false);
}}
@ -413,10 +398,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
title="Custom timezone"
placeholder="Etc/UTC"
maxLength={50}
onSubmit={tz => onQueryChange(query.changeQueryContext(changeTimezone(queryContext, tz)))}
onSubmit={tz => changeQueryContext(changeTimezone(queryContext, tz))}
onClose={() => setCustomTimezoneDialogOpen(false)}
/>
)}
{indexSpecDialogSpec && (
<IndexSpecDialog
onClose={() => setIndexSpecDialogSpec(undefined)}
onSave={indexSpec => changeQueryContext({ ...queryContext, indexSpec })}
indexSpec={indexSpecDialogSpec}
/>
)}
</div>
);
});