From cffa3bd263d77c48019fadc313db506ffa3ba24c Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 27 Sep 2022 23:46:48 -0700 Subject: [PATCH] [backport] Backport 24.0.1 web console issue fixes (#13146) * fix number of expected functions (#13050) * default to no compare (#13041) * quote columns, datasources in auto complete if needed (#13060) * Web console: better detection for arrays containing objects (#13077) * better detection for arrays containing objects * include boolean also * link to error docs (#13094) * Web console: correctly escape path based flatten specs (#13105) * fix path generation * do escape * fix replace * fix replace for good * append to exisitng callout (#13130) * better spec conversion with issues (#13136) * bump version to 24.0.1 --- web-console/package-lock.json | 2 +- web-console/package.json | 2 +- web-console/script/create-sql-docs.js | 8 +- web-console/src/ace-modes/dsql.js | 12 +- web-console/src/bootstrap/ace.scss | 12 + .../__snapshots__/header-bar.spec.tsx.snap | 2 +- .../__snapshots__/table-cell.spec.tsx.snap | 8 + .../components/table-cell/table-cell.spec.tsx | 7 + .../src/components/table-cell/table-cell.tsx | 3 +- .../table-filterable-cell.tsx | 6 +- ...inator-dynamic-config-dialog.spec.tsx.snap | 2 +- ...erload-dynamic-config-dialog.spec.tsx.snap | 2 +- .../retention-dialog.spec.tsx.snap | 2 +- .../flatten-spec/flatten-spec.spec.ts | 24 +- .../flatten-spec/flatten-spec.tsx | 24 +- .../ingestion-spec/ingestion-spec.spec.ts | 9 + .../ingestion-spec/ingestion-spec.tsx | 5 +- .../workbench-query/workbench-query.spec.ts | 14 + .../workbench-query/workbench-query.ts | 9 +- .../src/helpers/spec-conversion.spec.ts | 258 ++++++++++++++++++ web-console/src/helpers/spec-conversion.ts | 91 +++--- web-console/src/links.ts | 5 +- .../src/react-table/react-table-inputs.tsx | 4 +- web-console/src/utils/general.tsx | 10 + .../views/load-data-view/info-messages.tsx | 48 +++- .../views/load-data-view/load-data-view.tsx | 9 +- .../query-view/query-input/query-input.tsx | 7 +- .../__snapshots__/segments-view.spec.tsx.snap | 3 +- .../src/views/segments-view/segments-view.tsx | 11 +- .../execution-error-pane.spec.tsx.snap | 7 +- .../execution-error-pane.tsx | 10 +- .../flexible-query-input.tsx | 7 +- .../result-table-pane/result-table-pane.tsx | 2 +- web-console/unified-console.html | 2 +- 34 files changed, 511 insertions(+), 116 deletions(-) diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 6ac0e3a2cd9..e8609e31f4c 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -1,6 +1,6 @@ { "name": "web-console", - "version": "24.0.0", + "version": "24.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/web-console/package.json b/web-console/package.json index 0158b3bd620..241cafca6b1 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -1,6 +1,6 @@ { "name": "web-console", - "version": "24.0.0", + "version": "24.0.1", "description": "A web console for Apache Druid", "author": "Apache Druid Developers ", "license": "Apache-2.0", diff --git a/web-console/script/create-sql-docs.js b/web-console/script/create-sql-docs.js index 4258a0e99e4..6af65006f8e 100755 --- a/web-console/script/create-sql-docs.js +++ b/web-console/script/create-sql-docs.js @@ -23,7 +23,7 @@ const snarkdown = require('snarkdown'); const writefile = 'lib/sql-docs.js'; -const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 158; +const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162; const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14; function hasHtmlTags(str) { @@ -90,15 +90,15 @@ const readDoc = async () => { // Make sure there are enough functions found const numFunction = Object.keys(functionDocs).length; - if (numFunction < MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS) { + if (!(MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS <= numFunction)) { throw new Error( `Did not find enough function entries did the structure of '${readfile}' change? (found ${numFunction} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS})`, ); } // Make sure there are at least 10 data types for sanity - const numDataTypes = dataTypeDocs.length; - if (numDataTypes < MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES) { + const numDataTypes = Object.keys(dataTypeDocs).length; + if (!(MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES <= numDataTypes)) { throw new Error( `Did not find enough data type entries did the structure of '${readfile}' change? (found ${numDataTypes} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES})`, ); diff --git a/web-console/src/ace-modes/dsql.js b/web-console/src/ace-modes/dsql.js index c54970b17c6..3c743b7c495 100644 --- a/web-console/src/ace-modes/dsql.js +++ b/web-console/src/ace-modes/dsql.js @@ -63,6 +63,10 @@ ace.define( this.$rules = { start: [ + { + token: 'comment.issue', + regex: '--:ISSUE:.*$', + }, { token: 'comment', regex: '--.*$', @@ -73,17 +77,13 @@ ace.define( end: '\\*/', }, { - token: 'string', // " string + token: 'variable.column', // " quoted reference regex: '".*?"', }, { - token: 'string', // ' string + token: 'string', // ' string literal regex: "'.*?'", }, - { - token: 'string', // ` string (apache drill) - regex: '`.*?`', - }, { token: 'constant.numeric', // float regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b', diff --git a/web-console/src/bootstrap/ace.scss b/web-console/src/bootstrap/ace.scss index ebe74d4c412..e764348a8e3 100644 --- a/web-console/src/bootstrap/ace.scss +++ b/web-console/src/bootstrap/ace.scss @@ -25,6 +25,18 @@ .ace-solarized-dark { background-color: rgba($dark-gray1, 0.5); + // START: Custom code styles + .ace_variable.ace_column { + color: #2ceefb; + } + + .ace_comment.ace_issue { + color: #cb3116; + text-decoration: underline; + text-decoration-style: wavy; + } + // END: Custom code styles + &.no-background { background-color: transparent; } diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index 08619b24276..66b902596cf 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -326,7 +326,7 @@ exports[`HeaderBar matches snapshot 1`] = ` `; +exports[`TableCell matches snapshot array mixed 1`] = ` +
+ ["a",{"v":"b"},"c"] +
+`; + exports[`TableCell matches snapshot array short 1`] = `
{ expect(container.firstChild).toMatchSnapshot(); }); + it('matches snapshot array mixed', () => { + const tableCell = ; + + const { container } = render(tableCell); + expect(container.firstChild).toMatchSnapshot(); + }); + it('matches snapshot object', () => { const tableCell = ; diff --git a/web-console/src/components/table-cell/table-cell.tsx b/web-console/src/components/table-cell/table-cell.tsx index 3895a805a70..78f080b306d 100644 --- a/web-console/src/components/table-cell/table-cell.tsx +++ b/web-console/src/components/table-cell/table-cell.tsx @@ -21,6 +21,7 @@ import * as JSONBig from 'json-bigint-native'; import React, { useState } from 'react'; import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog'; +import { isSimpleArray } from '../../utils'; import { ActionIcon } from '../action-icon/action-icon'; import './table-cell.scss'; @@ -97,7 +98,7 @@ export const TableCell = React.memo(function TableCell(props: TableCellProps) { {isNaN(dateValue) ? 'Unusable date' : value.toISOString()}
); - } else if (Array.isArray(value)) { + } else if (isSimpleArray(value)) { return renderTruncated(`[${value.join(', ')}]`); } else if (typeof value === 'object') { return renderTruncated(JSONBig.stringify(value)); diff --git a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx index d3387fd37e8..14f568013f0 100644 --- a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx +++ b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx @@ -34,14 +34,14 @@ export interface TableFilterableCellProps { value: string; filters: Filter[]; onFiltersChange(filters: Filter[]): void; - disableComparisons?: boolean; + enableComparisons?: boolean; children?: ReactNode; } export const TableFilterableCell = React.memo(function TableFilterableCell( props: TableFilterableCellProps, ) { - const { field, value, children, filters, disableComparisons, onFiltersChange } = props; + const { field, value, children, filters, enableComparisons, onFiltersChange } = props; return ( ( - {(disableComparisons ? FILTER_MODES_NO_COMPARISONS : FILTER_MODES).map((mode, i) => ( + {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISONS).map((mode, i) => ( documentation diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap index 14d240ec01e..ac6e6eaf8a4 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap @@ -11,7 +11,7 @@ exports[`OverlordDynamicConfigDialog matches snapshot 1`] = ` Edit the overlord dynamic configuration on the fly. For more information please refer to the documentation diff --git a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap index c8b55ae5940..668128421ac 100644 --- a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap @@ -63,7 +63,7 @@ exports[`RetentionDialog matches snapshot 1`] = ` Druid uses rules to determine what data should be retained in the cluster. The rules are evaluated in order from top to bottom. For more information please refer to the diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index a35c1853810..822dd48eba3 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -22,7 +22,7 @@ describe('flatten-spec', () => { describe('computeFlattenExprsForData', () => { const data = [ { - context: { host: 'cla', topic: 'moon', bonus: { foo: 'bar' } }, + context: { host: 'cla', topic: 'moon', bonus: { 'fo.o': 'bar' } }, tags: ['a', 'b', 'c'], messages: [ { metric: 'request/time', value: 122 }, @@ -32,7 +32,7 @@ describe('flatten-spec', () => { value: 5, }, { - context: { host: 'piv', popic: 'sun' }, + context: { 'host': 'piv', '1pic': 'sun' }, tags: ['a', 'd'], messages: [ { metric: 'request/time', value: 44 }, @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { host: 'imp', dopik: 'fun' }, + context: { 'host': 'imp', "d\\o\npi'c'": 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -53,22 +53,12 @@ describe('flatten-spec', () => { ]; it('works for path, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'path', 'ignore-arrays')).toEqual([ - '$.context.bonus.foo', - '$.context.dopik', + expect(computeFlattenExprsForData(data, 'ignore-arrays')).toEqual([ + "$.context.bonus['fo.o']", '$.context.host', - '$.context.popic', '$.context.topic', - ]); - }); - - it('works for jq, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'jq', 'ignore-arrays')).toEqual([ - '.context.bonus.foo', - '.context.dopik', - '.context.host', - '.context.popic', - '.context.topic', + "$.context['1pic']", + "$.context['d\\\\o\npi\\'c\\'']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index 3845a83916d..5cf1d7c06c6 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -61,18 +61,22 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ }, ]; -export type ExprType = 'path' | 'jq'; export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; +function escapePathKey(pathKey: string): string { + return /^[a-z]\w*$/i.test(pathKey) + ? `.${pathKey}` + : `['${pathKey.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}']`; +} + export function computeFlattenPathsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, ): FlattenField[] { - return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => { + return computeFlattenExprsForData(data, arrayHandling).map(expr => { return { - name: expr.replace(/^\$?\./, ''), - type: exprType, + name: expr.replace(/^\$\./, '').replace(/['\]]/g, '').replace(/\[/g, '.'), + type: 'path', expr, }; }); @@ -80,7 +84,6 @@ export function computeFlattenPathsForData( export function computeFlattenExprsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, includeTopLevel = false, ): string[] { @@ -91,12 +94,7 @@ export function computeFlattenExprsForData( for (const datumKey of datumKeys) { const datumValue = datum[datumKey]; if (includeTopLevel || isNested(datumValue)) { - addPath( - seenPaths, - exprType === 'path' ? `$.${datumKey}` : `.${datumKey}`, - datumValue, - arrayHandling, - ); + addPath(seenPaths, `$${escapePathKey(datumKey)}`, datumValue, arrayHandling); } } } @@ -114,7 +112,7 @@ function addPath( if (!Array.isArray(value)) { const valueKeys = Object.keys(value); for (const valueKey of valueKeys) { - addPath(paths, `${path}.${valueKey}`, value[valueKey], arrayHandling); + addPath(paths, `${path}${escapePathKey(valueKey)}`, value[valueKey], arrayHandling); } } else if (arrayHandling === 'include-arrays') { for (let i = 0; i < value.length; i++) { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts index a846d29a818..3f4be422c79 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts @@ -725,6 +725,15 @@ describe('spec utils', () => { it('works for multi-value', () => { expect(guessColumnTypeFromInput(['a', ['b'], 'c'], false)).toEqual('string'); expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string'); + expect(guessColumnTypeFromInput([true, [true, 7, false], false, 'x'], false)).toEqual( + 'string', + ); + }); + + it('works for complex arrays', () => { + expect(guessColumnTypeFromInput([{ type: 'Dogs' }, { type: 'JavaScript' }], false)).toEqual( + 'COMPLEX', + ); }); it('works for strange json', () => { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 4ce36c655dc..f7492d35501 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -32,6 +32,7 @@ import { EMPTY_ARRAY, EMPTY_OBJECT, filterMap, + isSimpleArray, oneOf, parseCsvLine, typeIs, @@ -2309,7 +2310,7 @@ export function guessIsArrayFromHeaderAndRows( headerAndRows: SampleHeaderAndRows, column: string, ): boolean { - return headerAndRows.rows.some(r => Array.isArray(r.input?.[column])); + return headerAndRows.rows.some(r => isSimpleArray(r.input?.[column])); } export function guessColumnTypeFromInput( @@ -2322,7 +2323,7 @@ export function guessColumnTypeFromInput( if (!definedValues.length) return 'string'; // If we see any arrays in the input this is a multi-value dimension that must be a string - if (definedValues.some(v => Array.isArray(v))) return 'string'; + if (definedValues.some(v => isSimpleArray(v))) return 'string'; // If we see any JSON objects in the input assume COMPLEX if (definedValues.some(v => v && typeof v === 'object')) return 'COMPLEX'; diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 008947994f5..8f6f2581e1d 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => { sqlPrefixLines: 0, }); }); + + it('works with sql with ISSUE comment', () => { + const sql = sane` + SELECT * + --:ISSUE: There is something wrong with this query. + FROM wikipedia + `; + + const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql); + + expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow( + `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, + ); + }); }); describe('#getIngestDatasource', () => { diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 4200874aaf5..d30ea7d4cf9 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -576,10 +576,15 @@ export class WorkbenchQuery { apiQuery.query = queryPrepend + apiQuery.query + queryAppend; } - const m = /(--:context\s.+)(?:\n|$)/.exec(apiQuery.query); + const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query); if (m) { throw new Error( - `This query contains a context comment '${m[1]}'. Context comments have been deprecated. Please rewrite the context comment as a context parameter. The context parameter editor is located in the "Engine" dropdown.`, + `This query contains an ISSUE comment: ${m[1] + .trim() + .replace( + /\.$/, + '', + )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, ); } diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 9b1e421e939..43e7d31fac5 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -449,4 +449,262 @@ describe('spec conversion', () => { finalizeAggregations: false, }); }); + + it('converts with issue when there is a __time transform', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: '__time', expression: '_some_time_parse_expression_' }], + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + "comment", + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); + + it('converts with issue when there is a dimension transform and strange filter', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: 'comment', expression: '_some_expression_' }], + filter: { + type: 'strange', + }, + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index 498c908595b..9eedcce9f79 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -102,45 +102,55 @@ export function convertSpecToSql(spec: any): QueryWithContext { ); } + const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; + if (!Array.isArray(transforms)) { + throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); + } + let timeExpression: string; const column = timestampSpec.column || 'timestamp'; const columnRef = SqlRef.column(column); const format = timestampSpec.format || 'auto'; - switch (format) { - case 'auto': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; - break; + const timeTransform = transforms.find(t => t.name === '__time'); + if (timeTransform) { + timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`; + } else { + switch (format) { + case 'auto': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; + break; - case 'iso': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef})`; - break; + case 'iso': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef})`; + break; - case 'posix': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; - break; + case 'posix': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; + break; - case 'millis': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; - break; + case 'millis': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; + break; - case 'micro': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; - break; + case 'micro': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; + break; - case 'nano': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; - break; + case 'nano': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; + break; - default: - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; - break; + default: + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; + break; + } } if (timestampSpec.missingValue) { @@ -238,19 +248,24 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`SELECT`); - const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; - if (!Array.isArray(transforms)) - throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); if (transforms.length) { - lines.push(` -- The spec contained transforms that could not be automatically converted.`); + lines.push( + ` --:ISSUE: The spec contained transforms that could not be automatically converted.`, + ); } - const dimensionExpressions = [` ${timeExpression} AS __time,`].concat( + const dimensionExpressions = [ + ` ${timeExpression} AS __time,${ + timeTransform ? ` --:ISSUE: Transform for __time could not be converted` : '' + }`, + ].concat( dimensions.flatMap((dimension: DimensionSpec) => { const dimensionName = dimension.name; const relevantTransform = transforms.find(t => t.name === dimensionName); - return ` ${SqlRef.columnWithQuotes(dimensionName)},${ - relevantTransform ? ` -- Relevant transform: ${JSONBig.stringify(relevantTransform)}` : '' + return ` ${ + relevantTransform ? `REWRITE_[${relevantTransform.expression}]_TO_SQL AS ` : '' + }${SqlRef.columnWithQuotes(dimensionName)},${ + relevantTransform ? ` --:ISSUE: Transform for dimension could not be converted` : '' }`; }), ); @@ -275,9 +290,9 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`WHERE ${convertFilter(filter)}`); } catch { lines.push( - `-- The spec contained a filter that could not be automatically converted: ${JSONBig.stringify( + `WHERE REWRITE_[${JSONBig.stringify( filter, - )}`, + )}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually`, ); } } diff --git a/web-console/src/links.ts b/web-console/src/links.ts index d083a517c15..096b2f3eed9 100644 --- a/web-console/src/links.ts +++ b/web-console/src/links.ts @@ -19,7 +19,7 @@ import hasOwnProp from 'has-own-prop'; // This is set to the latest available version and should be updated to the next version before release -const DRUID_DOCS_VERSION = '24.0.0'; +const DRUID_DOCS_VERSION = '24.0.1'; function fillVersion(str: string): string { return str.replace(/\{\{VERSION}}/g, DRUID_DOCS_VERSION); @@ -63,6 +63,7 @@ export type LinkNames = | 'DOCS_SQL' | 'DOCS_RUNE' | 'DOCS_API' + | 'DOCS_MSQ_ERROR' | 'COMMUNITY' | 'SLACK' | 'USER_GROUP' @@ -82,6 +83,8 @@ export function getLink(linkName: LinkNames): string { return `${links.docsHref}/querying/querying.html`; case 'DOCS_API': return `${links.docsHref}/operations/api-reference.html`; + case 'DOCS_MSQ_ERROR': + return `${links.docsHref}/multi-stage-query/concepts.html#error-codes`; case 'COMMUNITY': return links.communityHref; case 'SLACK': diff --git a/web-console/src/react-table/react-table-inputs.tsx b/web-console/src/react-table/react-table-inputs.tsx index d7e02c3dc37..7dbb69b18f3 100644 --- a/web-console/src/react-table/react-table-inputs.tsx +++ b/web-console/src/react-table/react-table-inputs.tsx @@ -43,7 +43,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend const [menuOpen, setMenuOpen] = useState(false); const [focused, setFocused] = useState(false); - const disableComparisons = String(column.headerClassName).includes('disable-comparisons'); + const enableComparisons = String(column.headerClassName).includes('enable-comparisons'); const { mode, needle } = (filter ? parseFilterModeAndNeedle(filter, true) : undefined) || { mode: '~', @@ -64,7 +64,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend onInteraction={setMenuOpen} content={ - {(disableComparisons ? FILTER_MODES_NO_COMPARISON : FILTER_MODES).map((m, i) => ( + {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISON).map((m, i) => ( { + const t = typeof x; + return t === 'string' || t === 'number' || t === 'boolean'; + }) + ); +} + export function wait(ms: number): Promise { return new Promise(resolve => { setTimeout(resolve, ms); diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx index 7d7235bc3d3..2d151ea995f 100644 --- a/web-console/src/views/load-data-view/info-messages.tsx +++ b/web-console/src/views/load-data-view/info-messages.tsx @@ -16,12 +16,13 @@ * limitations under the License. */ -import { Callout, Code, FormGroup } from '@blueprintjs/core'; +import { Button, Callout, Code, FormGroup, Intent } from '@blueprintjs/core'; import React from 'react'; import { ExternalLink, LearnMore } from '../../components'; import { DimensionMode, getIngestionDocLink, IngestionSpec } from '../../druid-models'; import { getLink } from '../../links'; +import { deepGet, deepSet } from '../../utils'; export interface ConnectMessageProps { inlineMode: boolean; @@ -216,3 +217,48 @@ export const SpecMessage = React.memo(function SpecMessage() { ); }); + +export interface AppendToExistingIssueProps { + spec: Partial; + onChangeSpec(newSpec: Partial): void; +} + +export const AppendToExistingIssue = React.memo(function AppendToExistingIssue( + props: AppendToExistingIssueProps, +) { + const { spec, onChangeSpec } = props; + + const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type'); + if ( + partitionsSpecType === 'dynamic' || + deepGet(spec, 'spec.ioConfig.appendToExisting') !== true + ) { + return null; + } + + const dynamicPartitionSpec = { + type: 'dynamic', + maxRowsPerSegment: + deepGet(spec, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment') || + deepGet(spec, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment'), + }; + + return ( + + +

+ Only dynamic partitioning supports appendToExisting: true. You + have currently selected {partitionsSpecType} partitioning. +

+ + + + ); +}); diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 7afc13aa750..e4322e34e5b 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -168,6 +168,7 @@ import { ExamplePicker } from './example-picker/example-picker'; import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table'; import { FormEditor } from './form-editor/form-editor'; import { + AppendToExistingIssue, ConnectMessage, FilterMessage, ParserMessage, @@ -1490,7 +1491,6 @@ export class LoadDataView extends React.PureComponent r.input), - 'path', 'ignore-arrays', ); } @@ -3003,6 +3003,7 @@ export class LoadDataView extends React.PureComponent {nonsensicalSingleDimPartitioningMessage} + {this.renderNextBar({ disabled: invalidPartitionConfig(spec), @@ -3096,8 +3097,8 @@ export class LoadDataView extends React.PureComponent - deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') === 'dynamic', + // appendToExisting can only be set on 'dynamic' portioning. + // We chose to show it always and instead have a specific message, separate from this form, to notify the user of the issue. info: ( <> Creates segments as additional shards of the latest version, effectively @@ -3166,6 +3167,7 @@ export class LoadDataView extends React.PureComponent
+
{this.renderNextBar({})} @@ -3234,6 +3236,7 @@ export class LoadDataView extends React.PureComponent{`There is an issue with the spec: ${issueWithSpec}`} )} +
{!isEmptyIngestionSpec(spec) && ( diff --git a/web-console/src/views/query-view/query-input/query-input.tsx b/web-console/src/views/query-view/query-input/query-input.tsx index d4fd3fe6ae3..9f85c9de08a 100644 --- a/web-console/src/views/query-view/query-input/query-input.tsx +++ b/web-console/src/views/query-view/query-input/query-input.tsx @@ -20,6 +20,7 @@ import { ResizeEntry } from '@blueprintjs/core'; import { ResizeSensor2 } from '@blueprintjs/popover2'; import type { Ace } from 'ace-builds'; import ace from 'ace-builds'; +import { SqlRef, SqlTableRef } from 'druid-query-toolkit'; import escape from 'lodash.escape'; import React from 'react'; import AceEditor from 'react-ace'; @@ -150,7 +151,7 @@ export class QueryInput extends React.PureComponent d.TABLE_SCHEMA)).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 10, meta: 'schema', })), @@ -159,7 +160,7 @@ export class QueryInput extends React.PureComponent (currentSchema ? d.TABLE_SCHEMA === currentSchema : true)) .map(d => d.TABLE_NAME), ).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 49, meta: 'datasource', })), @@ -172,7 +173,7 @@ export class QueryInput extends React.PureComponent d.COLUMN_NAME), ).map(v => ({ - value: v, + value: SqlRef.column(v).toString(), score: 50, meta: 'column', })), diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap index 6ac125bfdf4..5d273a28691 100755 --- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap +++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap @@ -169,6 +169,7 @@ exports[`SegmentsView matches snapshot 1`] = ` "accessor": "start", "defaultSortDesc": true, "filterable": true, + "headerClassName": "enable-comparisons", "show": true, "sortable": true, "width": 160, @@ -179,6 +180,7 @@ exports[`SegmentsView matches snapshot 1`] = ` "accessor": "end", "defaultSortDesc": true, "filterable": true, + "headerClassName": "enable-comparisons", "show": true, "sortable": true, "width": 160, @@ -206,7 +208,6 @@ exports[`SegmentsView matches snapshot 1`] = ` "Cell": [Function], "Header": "Shard type", "accessor": [Function], - "headerClassName": "disable-comparisons", "id": "shard_type", "show": true, "sortable": false, diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index b0327182be1..85d02796226 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -485,7 +485,7 @@ END AS "time_span"`, }); } - private renderFilterableCell(field: string, disableComparisons = false) { + private renderFilterableCell(field: string, enableComparisons = false) { const { segmentFilter } = this.state; return (row: { value: any }) => ( @@ -494,7 +494,7 @@ END AS "time_span"`, value={row.value} filters={segmentFilter} onFiltersChange={filters => this.setState({ segmentFilter: filters })} - disableComparisons={disableComparisons} + enableComparisons={enableComparisons} > {row.value} @@ -582,21 +582,23 @@ END AS "time_span"`, Header: 'Start', show: visibleColumns.shown('Start'), accessor: 'start', + headerClassName: 'enable-comparisons', width: 160, sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, - Cell: this.renderFilterableCell('start'), + Cell: this.renderFilterableCell('start', true), }, { Header: 'End', show: visibleColumns.shown('End'), accessor: 'end', + headerClassName: 'enable-comparisons', width: 160, sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, - Cell: this.renderFilterableCell('end'), + Cell: this.renderFilterableCell('end', true), }, { Header: 'Version', @@ -623,7 +625,6 @@ END AS "time_span"`, id: 'shard_type', width: 100, sortable: false, - headerClassName: 'disable-comparisons', accessor: d => { let v: any; try { diff --git a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap index cf8bccd618b..cf7ed02ec72 100644 --- a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap @@ -8,7 +8,12 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `

- TooManyWarnings: + + TooManyWarnings + + : Too many warnings of type CannotParseExternalData generated (max = 10)

diff --git a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx index da7e3173b35..b0368534cc9 100644 --- a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx +++ b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx @@ -20,9 +20,10 @@ import { Callout } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React, { useState } from 'react'; -import { ClickToCopy } from '../../../components'; +import { ClickToCopy, ExternalLink } from '../../../components'; import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog'; import { Execution } from '../../../druid-models'; +import { getLink } from '../../../links'; import { downloadQueryDetailArchive } from '../../../utils'; import './execution-error-pane.scss'; @@ -43,7 +44,12 @@ export const ExecutionErrorPane = React.memo(function ExecutionErrorPane( return (

- {error.errorCode && <>{`${error.errorCode}: `}} + {error.errorCode && ( + <> + {error.errorCode} + {': '} + + )} {error.errorMessage || (exceptionStackTrace || '').split('\n')[0]} {exceptionStackTrace && ( <> diff --git a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx index c83865cdb19..1559e7981c8 100644 --- a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx +++ b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx @@ -21,6 +21,7 @@ import { ResizeSensor2 } from '@blueprintjs/popover2'; import type { Ace } from 'ace-builds'; import ace from 'ace-builds'; import classNames from 'classnames'; +import { SqlRef, SqlTableRef } from 'druid-query-toolkit'; import escape from 'lodash.escape'; import React from 'react'; import AceEditor from 'react-ace'; @@ -163,7 +164,7 @@ export class FlexibleQueryInput extends React.PureComponent< ) { const completions = ([] as any[]).concat( uniq(columnMetadata.map(d => d.TABLE_SCHEMA)).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 10, meta: 'schema', })), @@ -172,7 +173,7 @@ export class FlexibleQueryInput extends React.PureComponent< .filter(d => (currentSchema ? d.TABLE_SCHEMA === currentSchema : true)) .map(d => d.TABLE_NAME), ).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 49, meta: 'datasource', })), @@ -185,7 +186,7 @@ export class FlexibleQueryInput extends React.PureComponent< ) .map(d => d.COLUMN_NAME), ).map(v => ({ - value: v, + value: SqlRef.column(v).toString(), score: 50, meta: 'column', })), diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index e4b78b5af73..ba8b11001a4 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -82,7 +82,7 @@ function jsonValue(ex: SqlExpression, path: string): SqlExpression { } function getJsonPaths(jsons: Record[]): string[] { - return ['$.'].concat(computeFlattenExprsForData(jsons, 'path', 'include-arrays', true)); + return ['$.'].concat(computeFlattenExprsForData(jsons, 'include-arrays', true)); } function isComparable(x: unknown): boolean { diff --git a/web-console/unified-console.html b/web-console/unified-console.html index 19fa9d10bf6..ab0f7173922 100644 --- a/web-console/unified-console.html +++ b/web-console/unified-console.html @@ -71,6 +71,6 @@ }; - +