diff --git a/web-console/src/components/auto-form/auto-form.scss b/web-console/src/components/auto-form/auto-form.scss index 5523f0f8173..36f8c34d92d 100644 --- a/web-console/src/components/auto-form/auto-form.scss +++ b/web-console/src/components/auto-form/auto-form.scss @@ -28,4 +28,8 @@ right: 0; } } + + .custom-input input { + cursor: pointer; + } } diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index 1e56ef2b725..146de61b627 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -16,7 +16,14 @@ * limitations under the License. */ -import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core'; +import { + Button, + ButtonGroup, + FormGroup, + InputGroup, + Intent, + NumericInput, +} from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React from 'react'; @@ -46,7 +53,8 @@ export interface Field { | 'boolean' | 'string-array' | 'json' - | 'interval'; + | 'interval' + | 'custom'; defaultValue?: any; emptyValue?: any; suggestions?: Functor; @@ -64,6 +72,13 @@ export interface Field { valueAdjustment?: (value: any) => any; adjustment?: (model: Partial) => Partial; issueWithValue?: (value: any) => string | undefined; + + customSummary?: (v: any) => string; + customDialog?: (o: { + value: any; + onValueChange: (v: any) => void; + onClose: () => void; + }) => JSX.Element; } interface ComputedFieldValues { @@ -84,6 +99,7 @@ export interface AutoFormProps { export interface AutoFormState { showMore: boolean; + customDialog?: JSX.Element; } export class AutoForm> extends React.PureComponent< @@ -395,6 +411,36 @@ export class AutoForm> extends React.PureComponent ); } + private renderCustomInput(field: Field): JSX.Element { + const { model } = this.props; + const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field); + const effectiveValue = modelValue || defaultValue; + + const onEdit = () => { + this.setState({ + customDialog: field.customDialog?.({ + value: effectiveValue, + onValueChange: v => this.fieldChange(field, v), + onClose: () => { + this.setState({ customDialog: undefined }); + }, + }), + }); + }; + + return ( + } + onClick={onEdit} + /> + ); + } + renderFieldInput(field: Field) { switch (field.type) { case 'number': @@ -413,6 +459,8 @@ export class AutoForm> extends React.PureComponent return this.renderJsonInput(field); case 'interval': return this.renderIntervalInput(field); + case 'custom': + return this.renderCustomInput(field); default: throw new Error(`unknown field type '${field.type}'`); } @@ -464,7 +512,7 @@ export class AutoForm> extends React.PureComponent render(): JSX.Element { const { fields, model, showCustom } = this.props; - const { showMore } = this.state; + const { showMore, customDialog } = this.state; let shouldShowMore = false; const shownFields = fields.filter(field => { @@ -489,6 +537,7 @@ export class AutoForm> extends React.PureComponent {model && shownFields.map(this.renderField)} {model && showCustom && showCustom(model) && this.renderCustom()} {shouldShowMore && this.renderMoreOrLess()} + {customDialog} ); } diff --git a/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap b/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap new file mode 100644 index 00000000000..57d989621b7 --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap @@ -0,0 +1,317 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`IndexSpecDialog matches snapshot with indexSpec 1`] = ` + + +
+ + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. + , + "label": "String dictionary encoding", + "name": "stringDictionaryEncoding.type", + "suggestions": Array [ + "utf8", + "frontCoded", + ], + "type": "string", + }, + Object { + "defaultValue": 4, + "defined": [Function], + "info": + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. + , + "label": "String dictionary encoding bucket size", + "max": 128, + "min": 1, + "name": "stringDictionaryEncoding.bucketSize", + "type": "number", + }, + Object { + "defaultValue": "roaring", + "info": + Compression format for bitmap indexes. + , + "label": "Bitmap type", + "name": "bitmap.type", + "suggestions": Array [ + "roaring", + "concise", + ], + "type": "string", + }, + Object { + "defaultValue": true, + "defined": [Function], + "info": + Controls whether or not run-length encoding will be used when it is determined to be more space-efficient. + , + "label": "Bitmap compress run on serialization", + "name": "bitmap.compressRunOnSerialization", + "type": "boolean", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for dimension columns. + , + "name": "dimensionCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "longs", + "info": + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. + + auto + + encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. + + longs + + stores the value as-is with 8 bytes each. + , + "name": "longEncoding", + "suggestions": Array [ + "longs", + "auto", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for primitive type metric columns. + , + "name": "metricCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format to use for nested column raw data. + , + "label": "JSON compression", + "name": "jsonCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + ] + } + model={ + Object { + "dimensionCompression": "lzf", + } + } + onChange={[Function]} + /> +
+
+
+ + +
+
+ +`; + +exports[`IndexSpecDialog matches snapshot without compactionConfig 1`] = ` + + +
+ + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. + , + "label": "String dictionary encoding", + "name": "stringDictionaryEncoding.type", + "suggestions": Array [ + "utf8", + "frontCoded", + ], + "type": "string", + }, + Object { + "defaultValue": 4, + "defined": [Function], + "info": + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. + , + "label": "String dictionary encoding bucket size", + "max": 128, + "min": 1, + "name": "stringDictionaryEncoding.bucketSize", + "type": "number", + }, + Object { + "defaultValue": "roaring", + "info": + Compression format for bitmap indexes. + , + "label": "Bitmap type", + "name": "bitmap.type", + "suggestions": Array [ + "roaring", + "concise", + ], + "type": "string", + }, + Object { + "defaultValue": true, + "defined": [Function], + "info": + Controls whether or not run-length encoding will be used when it is determined to be more space-efficient. + , + "label": "Bitmap compress run on serialization", + "name": "bitmap.compressRunOnSerialization", + "type": "boolean", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for dimension columns. + , + "name": "dimensionCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "longs", + "info": + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. + + auto + + encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. + + longs + + stores the value as-is with 8 bytes each. + , + "name": "longEncoding", + "suggestions": Array [ + "longs", + "auto", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for primitive type metric columns. + , + "name": "metricCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format to use for nested column raw data. + , + "label": "JSON compression", + "name": "jsonCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + ] + } + model={Object {}} + onChange={[Function]} + /> +
+
+
+ + +
+
+ +`; diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss new file mode 100644 index 00000000000..e7cc53ee47d --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../variables'; + +.index-spec-dialog { + &.#{$bp-ns}-dialog { + height: 70vh; + } + + .form-json-selector { + margin: 15px; + } + + .content { + margin: 0 15px 10px 0; + padding: 0 5px 0 15px; + flex: 1; + overflow: auto; + } +} diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx new file mode 100644 index 00000000000..68f7f56b885 --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { IndexSpecDialog } from './index-spec-dialog'; + +describe('IndexSpecDialog', () => { + it('matches snapshot without compactionConfig', () => { + const compactionDialog = shallow( + {}} onSave={() => {}} indexSpec={undefined} />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); + + it('matches snapshot with indexSpec', () => { + const compactionDialog = shallow( + {}} + onSave={() => {}} + indexSpec={{ + dimensionCompression: 'lzf', + }} + />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx new file mode 100644 index 00000000000..4c870df45af --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Classes, Dialog, Intent } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components'; +import { INDEX_SPEC_FIELDS, IndexSpec } from '../../druid-models'; + +import './index-spec-dialog.scss'; + +export interface IndexSpecDialogProps { + title?: string; + onClose: () => void; + onSave: (indexSpec: IndexSpec) => void; + indexSpec: IndexSpec | undefined; +} + +export const IndexSpecDialog = React.memo(function IndexSpecDialog(props: IndexSpecDialogProps) { + const { title, indexSpec, onSave, onClose } = props; + + const [currentTab, setCurrentTab] = useState('form'); + const [currentIndexSpec, setCurrentIndexSpec] = useState(indexSpec || {}); + const [jsonError, setJsonError] = useState(); + + const issueWithCurrentIndexSpec = AutoForm.issueWithModel(currentIndexSpec, INDEX_SPEC_FIELDS); + + return ( + + +
+ {currentTab === 'form' ? ( + setCurrentIndexSpec(m)} + /> + ) : ( + { + setCurrentIndexSpec(v); + setJsonError(undefined); + }} + onError={setJsonError} + issueWithValue={value => AutoForm.issueWithModel(value, INDEX_SPEC_FIELDS)} + height="100%" + /> + )} +
+
+
+
+
+ + ); +}); diff --git a/web-console/src/druid-models/index-spec/index-spec.tsx b/web-console/src/druid-models/index-spec/index-spec.tsx new file mode 100644 index 00000000000..1a424629966 --- /dev/null +++ b/web-console/src/druid-models/index-spec/index-spec.tsx @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { Field } from '../../components'; +import { deepGet } from '../../utils'; + +export interface IndexSpec { + bitmap?: Bitmap; + dimensionCompression?: string; + stringDictionaryEncoding?: { type: 'utf8' | 'frontCoded'; bucketSize: number }; + metricCompression?: string; + longEncoding?: string; + jsonCompression?: string; +} + +export interface Bitmap { + type: string; + compressRunOnSerialization?: boolean; +} + +export function summarizeIndexSpec(indexSpec: IndexSpec | undefined): string { + if (!indexSpec) return ''; + + const { stringDictionaryEncoding, bitmap, longEncoding } = indexSpec; + + const ret: string[] = []; + if (stringDictionaryEncoding) { + switch (stringDictionaryEncoding.type) { + case 'frontCoded': + ret.push(`frontCoded(${stringDictionaryEncoding.bucketSize || 4})`); + break; + + default: + ret.push(stringDictionaryEncoding.type); + break; + } + } + + if (bitmap) { + ret.push(bitmap.type); + } + + if (longEncoding) { + ret.push(longEncoding); + } + + return ret.join('; '); +} + +export const INDEX_SPEC_FIELDS: Field[] = [ + { + name: 'stringDictionaryEncoding.type', + label: 'String dictionary encoding', + type: 'string', + defaultValue: 'utf8', + suggestions: ['utf8', 'frontCoded'], + info: ( + <> + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> + columns. + + ), + }, + { + name: 'stringDictionaryEncoding.bucketSize', + label: 'String dictionary encoding bucket size', + type: 'number', + defaultValue: 4, + min: 1, + max: 128, + defined: spec => deepGet(spec, 'stringDictionaryEncoding.type') === 'frontCoded', + info: ( + <> + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, + maximum is 128. + + ), + }, + + { + name: 'bitmap.type', + label: 'Bitmap type', + type: 'string', + defaultValue: 'roaring', + suggestions: ['roaring', 'concise'], + info: <>Compression format for bitmap indexes., + }, + { + name: 'bitmap.compressRunOnSerialization', + label: 'Bitmap compress run on serialization', + type: 'boolean', + defaultValue: true, + defined: spec => (deepGet(spec, 'bitmap.type') || 'roaring') === 'roaring', + info: ( + <> + Controls whether or not run-length encoding will be used when it is determined to be more + space-efficient. + + ), + }, + + { + name: 'dimensionCompression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format for dimension columns., + }, + + { + name: 'longEncoding', + type: 'string', + defaultValue: 'longs', + suggestions: ['longs', 'auto'], + info: ( + <> + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or + metrics. auto encodes the values using offset or lookup table depending on + column cardinality, and store them with variable size. longs stores the value + as-is with 8 bytes each. + + ), + }, + { + name: 'metricCompression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format for primitive type metric columns., + }, + + { + name: 'jsonCompression', + label: 'JSON compression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format to use for nested column raw data. , + }, +]; diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts index 359ba70440b..0b4ad6b65f7 100644 --- a/web-console/src/druid-models/index.ts +++ b/web-console/src/druid-models/index.ts @@ -25,6 +25,7 @@ export * from './execution/execution'; export * from './external-config/external-config'; export * from './filter/filter'; export * from './flatten-spec/flatten-spec'; +export * from './index-spec/index-spec'; export * from './ingest-query-pattern/ingest-query-pattern'; export * from './ingestion-spec/ingestion-spec'; export * from './input-format/input-format'; diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index b68052abd47..f144f4c2ad2 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -21,6 +21,7 @@ import { range } from 'd3-array'; import React from 'react'; import { AutoForm, ExternalLink, Field } from '../../components'; +import { IndexSpecDialog } from '../../dialogs/index-spec-dialog/index-spec-dialog'; import { getLink } from '../../links'; import { allowKeys, @@ -44,6 +45,7 @@ import { getDimensionSpecs, getDimensionSpecType, } from '../dimension-spec/dimension-spec'; +import { IndexSpec, summarizeIndexSpec } from '../index-spec/index-spec'; import { InputFormat, issueWithInputFormat } from '../input-format/input-format'; import { FILTER_SUGGESTIONS, @@ -1379,6 +1381,7 @@ export interface TuningConfig { partitionsSpec?: PartitionsSpec; maxPendingPersists?: number; indexSpec?: IndexSpec; + indexSpecForIntermediatePersists?: IndexSpec; forceExtendableShardSpecs?: boolean; forceGuaranteedRollup?: boolean; reportParseExceptions?: boolean; @@ -1869,103 +1872,38 @@ const TUNING_FORM_FIELDS: Field[] = [ }, { - name: 'spec.tuningConfig.indexSpec.bitmap.type', - label: 'Index bitmap type', - type: 'string', - defaultValue: 'roaring', - suggestions: ['concise', 'roaring'], + name: 'spec.tuningConfig.indexSpec', + type: 'custom', hideInMore: true, - info: <>Compression format for bitmap indexes., + info: <>Defines segment storage format options to use at indexing time., + placeholder: 'Default index spec', + customSummary: summarizeIndexSpec, + customDialog: ({ value, onValueChange, onClose }) => ( + + ), }, { - name: 'spec.tuningConfig.indexSpec.bitmap.compressRunOnSerialization', - type: 'boolean', - defaultValue: true, - defined: spec => deepGet(spec, 'spec.tuningConfig.indexSpec.bitmap.type') === 'roaring', + name: 'spec.tuningConfig.indexSpecForIntermediatePersists', + type: 'custom', + hideInMore: true, info: ( <> - Controls whether or not run-length encoding will be used when it is determined to be more - space-efficient. + Defines segment storage format options to use at indexing time for intermediate persisted + temporary segments. ), + placeholder: 'Default index spec', + customSummary: summarizeIndexSpec, + customDialog: ({ value, onValueChange, onClose }) => ( + + ), }, - { - name: 'spec.tuningConfig.indexSpec.dimensionCompression', - label: 'Index dimension compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format for dimension columns., - }, - - { - name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type', - label: 'Index string dictionary encoding', - type: 'string', - defaultValue: 'utf8', - suggestions: ['utf8', 'frontCoded'], - hideInMore: true, - info: ( - <> - Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> - columns. - - ), - }, - { - name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.bucketSize', - label: 'Index string dictionary encoding bucket size', - type: 'number', - defaultValue: 4, - min: 1, - max: 128, - defined: spec => - deepGet(spec, 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type') === 'frontCoded', - hideInMore: true, - info: ( - <> - The number of values to place in a bucket to perform delta encoding. Must be a power of 2, - maximum is 128. - - ), - }, - - { - name: 'spec.tuningConfig.indexSpec.metricCompression', - label: 'Index metric compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format for primitive type metric columns., - }, - { - name: 'spec.tuningConfig.indexSpec.longEncoding', - label: 'Index long encoding', - type: 'string', - defaultValue: 'longs', - suggestions: ['longs', 'auto'], - hideInMore: true, - info: ( - <> - Encoding format for long-typed columns. Applies regardless of whether they are dimensions or - metrics. auto encodes the values using offset or lookup table depending on - column cardinality, and store them with variable size. longs stores the value - as-is with 8 bytes each. - - ), - }, - { - name: 'spec.tuningConfig.indexSpec.jsonCompression', - label: 'Index JSON compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format to use for nested column raw data. , - }, { name: 'spec.tuningConfig.splitHintSpec.maxSplitSize', type: 'number', @@ -2172,18 +2110,6 @@ export function getTuningFormFields() { return TUNING_FORM_FIELDS; } -export interface IndexSpec { - bitmap?: Bitmap; - dimensionCompression?: string; - metricCompression?: string; - longEncoding?: string; -} - -export interface Bitmap { - type: string; - compressRunOnSerialization?: boolean; -} - // -------------- export function updateIngestionType( diff --git a/web-console/src/druid-models/workbench-query/workbench-query-part.ts b/web-console/src/druid-models/workbench-query/workbench-query-part.ts index 604cfb01321..5e4afb453f0 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query-part.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query-part.ts @@ -62,10 +62,10 @@ export class WorkbenchQueryPart { static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined { // Assuming the queryFragment is no parsable find the prefix that look like: // REPLACEINTOSELECT - const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/)?.index; + const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index; if (typeof matchInsertReplaceIndex !== 'number') return; - const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/); + const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/i); const fragmentQuery = SqlQuery.maybeParse( queryFragment.substring(matchInsertReplaceIndex, matchEnd?.index) + ' SELECT * FROM t', ); diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 5d7e2615098..9af0fb24070 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -465,7 +465,7 @@ describe('WorkbenchQuery', () => { it('works with INSERT (unparsable)', () => { const sql = sane` -- Some comment - INSERT INTO trips2 + INSERT into trips2 SELECT TIME_PARSE(pickup_datetime) AS __time, * diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 0239da18541..2f6aa59f51e 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -106,6 +106,9 @@ describe('spec conversion', () => { partitionDimension: 'isRobot', targetRowsPerSegment: 150000, }, + indexSpec: { + dimensionCompression: 'lzf', + }, forceGuaranteedRollup: true, maxNumConcurrentSubTasks: 4, maxParseExceptions: 3, @@ -159,6 +162,9 @@ describe('spec conversion', () => { maxParseExceptions: 3, finalizeAggregations: false, maxNumTasks: 5, + indexSpec: { + dimensionCompression: 'lzf', + }, }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index d35433917f5..9b76e787dd6 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -70,6 +70,11 @@ export function convertSpecToSql(spec: any): QueryWithContext { groupByEnableMultiValueUnnesting: false, }; + const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec'); + if (indexSpec) { + context.indexSpec = indexSpec; + } + const lines: string[] = []; const rollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup') ?? true; diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index 572120c9e73..7299760b462 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -33,6 +33,7 @@ import React, { useCallback, useMemo, useState } from 'react'; import { MenuCheckbox, MenuTristate } from '../../../components'; import { EditContextDialog, StringInputDialog } from '../../../dialogs'; +import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog'; import { changeDurableShuffleStorage, changeFinalizeAggregations, @@ -51,9 +52,12 @@ import { getUseApproximateCountDistinct, getUseApproximateTopN, getUseCache, + IndexSpec, + QueryContext, + summarizeIndexSpec, WorkbenchQuery, } from '../../../druid-models'; -import { pluralIfNeeded, tickIcon } from '../../../utils'; +import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils'; import { MaxTasksButton } from '../max-tasks-button/max-tasks-button'; import './run-panel.scss'; @@ -94,6 +98,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines } = props; const [editContextDialogOpen, setEditContextDialogOpen] = useState(false); const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false); + const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState(); const emptyQuery = query.isEmptyQuery(); const ingestMode = query.isIngestQuery(); @@ -104,6 +109,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const finalizeAggregations = getFinalizeAggregations(queryContext); const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext); const durableShuffleStorage = getDurableShuffleStorage(queryContext); + const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec'); const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext); const useApproximateTopN = getUseApproximateTopN(queryContext); const useCache = getUseCache(queryContext); @@ -157,6 +163,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { ); } + function changeQueryContext(queryContext: QueryContext) { + onQueryChange(query.changeQueryContext(queryContext)); + } + const availableEngines = ([undefined] as (DruidEngine | undefined)[]).concat(queryEngines); function offsetOptions(): JSX.Element[] { @@ -170,9 +180,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(offset === timezone)} text={offset} shouldDismissPopover={false} - onClick={() => { - onQueryChange(query.changeQueryContext(changeTimezone(queryContext, offset))); - }} + onClick={() => changeQueryContext(changeTimezone(queryContext, offset))} />, ); } @@ -233,11 +241,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(!timezone)} text="Default" shouldDismissPopover={false} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeTimezone(queryContext, undefined)), - ); - }} + onClick={() => changeQueryContext(changeTimezone(queryContext, undefined))} /> {NAMED_TIMEZONES.map(namedTimezone => ( @@ -246,11 +250,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(namedTimezone === timezone)} text={namedTimezone} shouldDismissPopover={false} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeTimezone(queryContext, namedTimezone)), - ); - }} + onClick={() => + changeQueryContext(changeTimezone(queryContext, namedTimezone)) + } /> ))} @@ -276,11 +278,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { key={String(v)} icon={tickIcon(v === maxParseExceptions)} text={v === -1 ? '∞ (-1)' : String(v)} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeMaxParseExceptions(queryContext, v)), - ); - }} + onClick={() => + changeQueryContext(changeMaxParseExceptions(queryContext, v)) + } shouldDismissPopover={false} /> ))} @@ -290,35 +290,36 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { text="Finalize aggregations" value={finalizeAggregations} undefinedEffectiveValue={!ingestMode} - onValueChange={v => { - onQueryChange( - query.changeQueryContext(changeFinalizeAggregations(queryContext, v)), - ); - }} + onValueChange={v => + changeQueryContext(changeFinalizeAggregations(queryContext, v)) + } /> { - onQueryChange( - query.changeQueryContext( - changeGroupByEnableMultiValueUnnesting(queryContext, v), - ), - ); + onValueChange={v => + changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v)) + } + /> + { + setIndexSpecDialogSpec(indexSpec || {}); }} /> { - onQueryChange( - query.changeQueryContext( - changeDurableShuffleStorage(queryContext, !durableShuffleStorage), - ), - ); - }} + onChange={() => + changeQueryContext( + changeDurableShuffleStorage(queryContext, !durableShuffleStorage), + ) + } /> ) : ( @@ -326,22 +327,16 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { { - onQueryChange( - query.changeQueryContext(changeUseCache(queryContext, !useCache)), - ); - }} + onChange={() => changeQueryContext(changeUseCache(queryContext, !useCache))} /> { - onQueryChange( - query.changeQueryContext( - changeUseApproximateTopN(queryContext, !useApproximateTopN), - ), - ); - }} + onChange={() => + changeQueryContext( + changeUseApproximateTopN(queryContext, !useApproximateTopN), + ) + } /> )} @@ -349,16 +344,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { { - onQueryChange( - query.changeQueryContext( - changeUseApproximateCountDistinct( - queryContext, - !useApproximateCountDistinct, - ), + onChange={() => + changeQueryContext( + changeUseApproximateCountDistinct( + queryContext, + !useApproximateCountDistinct, ), - ); - }} + ) + } /> )} {effectiveEngine === 'sql-msq-task' && ( - - onQueryChange(query.changeQueryContext(queryContext)) - } - /> + )} )} @@ -399,10 +387,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { {editContextDialogOpen && ( { - if (!onQueryChange) return; - onQueryChange(query.changeQueryContext(newContext)); - }} + onQueryContextChange={changeQueryContext} onClose={() => { setEditContextDialogOpen(false); }} @@ -413,10 +398,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { title="Custom timezone" placeholder="Etc/UTC" maxLength={50} - onSubmit={tz => onQueryChange(query.changeQueryContext(changeTimezone(queryContext, tz)))} + onSubmit={tz => changeQueryContext(changeTimezone(queryContext, tz))} onClose={() => setCustomTimezoneDialogOpen(false)} /> )} + {indexSpecDialogSpec && ( + setIndexSpecDialogSpec(undefined)} + onSave={indexSpec => changeQueryContext({ ...queryContext, indexSpec })} + indexSpec={indexSpecDialogSpec} + /> + )} ); });