From dc2ae1e99c79f08d29fcfa7d38d95908a9fe3225 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 16 Aug 2023 23:50:43 -0700 Subject: [PATCH] Web console: improving the helper queries by allowing for running inline helper queries (#14801) * remove helper queries * fix tests * take care of zero queries also * switch to better place --- web-console/e2e-tests/tutorial-batch.spec.ts | 2 +- .../query-error-pane/query-error-pane.tsx | 6 +- web-console/src/druid-models/index.ts | 1 - .../workbench-query/workbench-query-part.ts | 261 ---------- .../workbench-query/workbench-query.spec.ts | 143 +----- .../workbench-query/workbench-query.ts | 404 +++++---------- .../src/singletons/ace-editor-state-cache.ts | 6 +- .../src/singletons/execution-state-cache.ts | 6 - .../singletons/workbench-running-promises.ts | 8 +- web-console/src/utils/druid-query.spec.ts | 39 +- web-console/src/utils/druid-query.ts | 38 +- web-console/src/utils/general.spec.ts | 21 +- web-console/src/utils/general.tsx | 14 +- web-console/src/utils/query-cursor.ts | 11 +- web-console/src/utils/sql.spec.ts | 349 +++++++++++++ web-console/src/utils/sql.ts | 81 ++- .../column-editor/column-editor.tsx | 1 - .../expression-editor-dialog.tsx | 1 - .../schema-step/schema-step.tsx | 6 +- .../src/views/workbench-view/demo-queries.ts | 2 +- .../execution-details-pane.tsx | 1 - .../flexible-query-input.spec.tsx.snap | 2 +- .../flexible-query-input.scss | 52 ++ .../flexible-query-input.spec.tsx | 6 +- .../flexible-query-input.tsx | 167 +++++-- .../helper-query/helper-query.scss | 100 ---- .../helper-query/helper-query.tsx | 473 ------------------ .../workbench-view/query-tab/query-tab.scss | 41 +- .../workbench-view/query-tab/query-tab.tsx | 123 ++--- .../workbench-view/run-panel/run-panel.tsx | 8 +- .../views/workbench-view/workbench-view.tsx | 11 +- 31 files changed, 904 insertions(+), 1480 deletions(-) delete mode 100644 web-console/src/druid-models/workbench-query/workbench-query-part.ts create mode 100644 web-console/src/utils/sql.spec.ts delete mode 100644 web-console/src/views/workbench-view/helper-query/helper-query.scss delete mode 100644 web-console/src/views/workbench-view/helper-query/helper-query.tsx diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index daae46a60fb..4b4d90e200b 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -36,7 +36,7 @@ import { waitTillWebConsoleReady } from './util/setup'; jest.setTimeout(5 * 60 * 1000); -const ALL_SORTS_OF_CHARS = '<>|!@#$%^&`\'".,:;\\*()[]{}Россия 한국 中国!?~'; +const ALL_SORTS_OF_CHARS = '<>|!@#$%^&`\'".,:;\\*()[]{}Україна 한국 中国!?~'; describe('Tutorial: Loading a file', () => { let browser: playwright.Browser; diff --git a/web-console/src/components/query-error-pane/query-error-pane.tsx b/web-console/src/components/query-error-pane/query-error-pane.tsx index f8e0d3a622c..284b58e21ff 100644 --- a/web-console/src/components/query-error-pane/query-error-pane.tsx +++ b/web-console/src/components/query-error-pane/query-error-pane.tsx @@ -39,7 +39,7 @@ export const QueryErrorPane = React.memo(function QueryErrorPane(props: QueryErr return
{error.message}
; } - const { position, suggestion } = error; + const { startRowColumn, suggestion } = error; let suggestionElement: JSX.Element | undefined; if (suggestion && queryString && onQueryStringChange) { const newQuery = suggestion.fn(queryString); @@ -69,14 +69,14 @@ export const QueryErrorPane = React.memo(function QueryErrorPane(props: QueryErr )} {error.errorMessageWithoutExpectation && (

- {position ? ( + {startRowColumn ? ( ( { - moveCursorTo(position); + moveCursorTo(startRowColumn); }} > {found} diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts index 16edb184fcb..18cd812c610 100644 --- a/web-console/src/druid-models/index.ts +++ b/web-console/src/druid-models/index.ts @@ -41,4 +41,3 @@ export * from './time/time'; export * from './timestamp-spec/timestamp-spec'; export * from './transform-spec/transform-spec'; export * from './workbench-query/workbench-query'; -export * from './workbench-query/workbench-query-part'; diff --git a/web-console/src/druid-models/workbench-query/workbench-query-part.ts b/web-console/src/druid-models/workbench-query/workbench-query-part.ts deleted file mode 100644 index 45fce5914b4..00000000000 --- a/web-console/src/druid-models/workbench-query/workbench-query-part.ts +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { SqlValues, SqlWithQuery } from '@druid-toolkit/query'; -import { SqlExpression, SqlQuery, T } from '@druid-toolkit/query'; -import Hjson from 'hjson'; -import * as JSONBig from 'json-bigint-native'; - -import type { ColumnMetadata } from '../../utils'; -import { compact, filterMap, generate8HexId } from '../../utils'; -import type { LastExecution } from '../execution/execution'; -import { validateLastExecution } from '../execution/execution'; -import { fitExternalConfigPattern } from '../external-config/external-config'; - -// ----------------------------- - -export interface WorkbenchQueryPartValue { - id: string; - queryName?: string; - queryString: string; - collapsed?: boolean; - lastExecution?: LastExecution; -} - -export class WorkbenchQueryPart { - static blank() { - return new WorkbenchQueryPart({ - id: generate8HexId(), - queryString: '', - }); - } - - static fromQuery(query: SqlQuery | SqlValues, queryName?: string, collapsed?: boolean) { - return this.fromQueryString(query.changeParens([]).toString(), queryName, collapsed); - } - - static fromQueryString(queryString: string, queryName?: string, collapsed?: boolean) { - return new WorkbenchQueryPart({ - id: generate8HexId(), - queryName, - queryString, - collapsed, - }); - } - - static isTaskEngineNeeded(queryString: string): boolean { - return /EXTERN\s*\(|(?:INSERT|REPLACE)\s+INTO/im.test(queryString); - } - - static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined { - // Assuming the queryFragment is no parsable find the prefix that look like: - // REPLACEINTOSELECT - const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index; - if (typeof matchInsertReplaceIndex !== 'number') return; - - const queryStartingWithInsertOrReplace = queryFragment.substring(matchInsertReplaceIndex); - - const matchEnd = queryStartingWithInsertOrReplace.match(/\b(?:SELECT|WITH)\b|$/i); - const fragmentQuery = SqlQuery.maybeParse( - queryStartingWithInsertOrReplace.substring(0, matchEnd?.index) + ' SELECT * FROM t', - ); - if (!fragmentQuery) return; - - return fragmentQuery.getIngestTable()?.getName(); - } - - public readonly id: string; - public readonly queryName?: string; - public readonly queryString: string; - public readonly collapsed: boolean; - public readonly lastExecution?: LastExecution; - - public readonly parsedQuery?: SqlQuery; - - constructor(value: WorkbenchQueryPartValue) { - this.id = value.id; - this.queryName = value.queryName; - this.queryString = value.queryString; - this.collapsed = Boolean(value.collapsed); - this.lastExecution = validateLastExecution(value.lastExecution); - - try { - this.parsedQuery = SqlQuery.parse(this.queryString); - } catch {} - } - - public valueOf(): WorkbenchQueryPartValue { - return { - id: this.id, - queryName: this.queryName, - queryString: this.queryString, - collapsed: this.collapsed, - lastExecution: this.lastExecution, - }; - } - - public changeId(id: string): WorkbenchQueryPart { - return new WorkbenchQueryPart({ ...this.valueOf(), id }); - } - - public changeQueryName(queryName: string): WorkbenchQueryPart { - return new WorkbenchQueryPart({ ...this.valueOf(), queryName }); - } - - public changeQueryString(queryString: string): WorkbenchQueryPart { - return new WorkbenchQueryPart({ ...this.valueOf(), queryString }); - } - - public changeCollapsed(collapsed: boolean): WorkbenchQueryPart { - return new WorkbenchQueryPart({ ...this.valueOf(), collapsed }); - } - - public changeLastExecution(lastExecution: LastExecution | undefined): WorkbenchQueryPart { - return new WorkbenchQueryPart({ ...this.valueOf(), lastExecution }); - } - - public clear(): WorkbenchQueryPart { - return new WorkbenchQueryPart({ - ...this.valueOf(), - queryString: '', - }); - } - - public isEmptyQuery(): boolean { - return this.queryString.trim() === ''; - } - - public isJsonLike(): boolean { - return this.queryString.trim().startsWith('{'); - } - - public issueWithJson(): string | undefined { - try { - Hjson.parse(this.queryString); - } catch (e) { - return e.message; - } - return; - } - - public isSqlInJson(): boolean { - try { - const query = Hjson.parse(this.queryString); - return typeof query.query === 'string'; - } catch { - return false; - } - } - - public getSqlString(): string { - if (this.isJsonLike()) { - const query = Hjson.parse(this.queryString); - return typeof query.query === 'string' ? query.query : ''; - } else { - return this.queryString; - } - } - - public prettyPrintJson(): WorkbenchQueryPart { - let parsed: unknown; - try { - parsed = Hjson.parse(this.queryString); - } catch { - return this; - } - return this.changeQueryString(JSONBig.stringify(parsed, undefined, 2)); - } - - public getIngestDatasource(): string | undefined { - const { queryString, parsedQuery } = this; - if (parsedQuery) { - return parsedQuery.getIngestTable()?.getName(); - } - - if (this.isJsonLike()) return; - - return WorkbenchQueryPart.getIngestDatasourceFromQueryFragment(queryString); - } - - public getInlineMetadata(): ColumnMetadata[] { - const { queryName, parsedQuery } = this; - if (queryName && parsedQuery) { - try { - return fitExternalConfigPattern(parsedQuery).signature.map(columnDeclaration => ({ - COLUMN_NAME: columnDeclaration.getColumnName(), - DATA_TYPE: columnDeclaration.columnType.getEffectiveType(), - TABLE_NAME: queryName, - TABLE_SCHEMA: 'druid', - })); - } catch { - return filterMap(parsedQuery.getSelectExpressionsArray(), ex => { - const outputName = ex.getOutputName(); - if (!outputName) return; - return { - COLUMN_NAME: outputName, - DATA_TYPE: 'UNKNOWN', - TABLE_NAME: queryName, - TABLE_SCHEMA: 'druid', - }; - }); - } - } - return []; - } - - public isTaskEngineNeeded(): boolean { - return WorkbenchQueryPart.isTaskEngineNeeded(this.queryString); - } - - public extractCteHelpers(): WorkbenchQueryPart[] | undefined { - let flatQuery: SqlQuery; - try { - // We need to do our own parsing here because this.parseQuery necessarily must be a SqlQuery - // object, and we might have a SqlWithQuery here. - flatQuery = (SqlExpression.parse(this.queryString) as SqlWithQuery).flattenWith(); - } catch { - return; - } - - const possibleNewParts = flatQuery.getWithParts().map(({ table, columns, query }) => { - if (columns) return; - return WorkbenchQueryPart.fromQuery(query, table.name, true); - }); - if (!possibleNewParts.length) return; - - const newParts = compact(possibleNewParts); - if (newParts.length !== possibleNewParts.length) return; - - return newParts.concat(this.changeQueryString(flatQuery.changeWithParts(undefined).toString())); - } - - public toWithPart(): string { - const { queryName, queryString } = this; - return `${T(queryName || 'q')} AS (\n${queryString}\n)`; - } - - public duplicate(): WorkbenchQueryPart { - return this.changeId(generate8HexId()).changeLastExecution(undefined); - } - - public addPreviewLimit(): WorkbenchQueryPart { - const { parsedQuery } = this; - if (!parsedQuery || parsedQuery.hasLimit()) return this; - return this.changeQueryString(parsedQuery.changeLimitValue(10000).toString()); - } -} diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index bb1fa9c9557..8732b93d424 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -19,7 +19,6 @@ import { sane } from '@druid-toolkit/query'; import { WorkbenchQuery } from './workbench-query'; -import { WorkbenchQueryPart } from './workbench-query-part'; describe('WorkbenchQuery', () => { beforeAll(() => { @@ -107,12 +106,6 @@ describe('WorkbenchQuery', () => { describe('.fromString', () => { const tabString = sane` - ===== Helper: q ===== - - SELECT * - - FROM wikipedia - ===== Query ===== SELECT * FROM q @@ -207,6 +200,7 @@ describe('WorkbenchQuery', () => { expect(apiQuery).toEqual({ cancelQueryId: 'deadbeef-9fb0-499c-8475-ea461e96a4fd', engine: 'native', + prefixLines: 0, query: { aggregations: [ { @@ -258,6 +252,7 @@ describe('WorkbenchQuery', () => { expect(apiQuery).toEqual({ cancelQueryId: 'lol', engine: 'native', + prefixLines: 0, query: { aggregations: [ { @@ -302,7 +297,7 @@ describe('WorkbenchQuery', () => { sqlTypesHeader: true, typesHeader: true, }, - sqlPrefixLines: 0, + prefixLines: 0, }); }); @@ -328,7 +323,7 @@ describe('WorkbenchQuery', () => { sqlTypesHeader: true, typesHeader: true, }, - sqlPrefixLines: 0, + prefixLines: 0, }); }); @@ -368,7 +363,7 @@ describe('WorkbenchQuery', () => { sqlTypesHeader: true, typesHeader: true, }, - sqlPrefixLines: 0, + prefixLines: 0, }); }); @@ -407,7 +402,7 @@ describe('WorkbenchQuery', () => { sqlTypesHeader: true, typesHeader: true, }, - sqlPrefixLines: 0, + prefixLines: 0, }); }); @@ -435,7 +430,7 @@ describe('WorkbenchQuery', () => { sqlTypesHeader: true, typesHeader: true, }, - sqlPrefixLines: 0, + prefixLines: 0, }); }); @@ -536,130 +531,6 @@ describe('WorkbenchQuery', () => { }); }); - describe('#extractCteHelpers', () => { - it('works', () => { - const sql = sane` - REPLACE INTO task_statuses OVERWRITE ALL - WITH - task_statuses AS ( - SELECT * FROM - TABLE( - EXTERN( - '{"type":"local","baseDir":"/Users/vadim/Desktop/","filter":"task_statuses.json"}', - '{"type":"json"}', - '[{"name":"id","type":"string"},{"name":"status","type":"string"},{"name":"duration","type":"long"},{"name":"errorMsg","type":"string"},{"name":"created_date","type":"string"}]' - ) - ) - ) - ( - --PLACE INTO task_statuses OVERWRITE ALL - SELECT - id, - status, - duration, - errorMsg, - created_date - FROM task_statuses - --RTITIONED BY ALL - ) - PARTITIONED BY ALL - `; - - expect(WorkbenchQuery.blank().changeQueryString(sql).extractCteHelpers().getQueryString()) - .toEqual(sane` - REPLACE INTO task_statuses OVERWRITE ALL - SELECT - id, - status, - duration, - errorMsg, - created_date - FROM task_statuses - PARTITIONED BY ALL - `); - }); - }); - - describe('#materializeHelpers', () => { - it('works', () => { - expect( - WorkbenchQuery.blank() - .changeQueryParts([ - new WorkbenchQueryPart({ - id: 'aaa', - queryName: 'kttm_data', - queryString: sane` - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}' - ) - ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) - `, - }), - new WorkbenchQueryPart({ - id: 'bbb', - queryName: 'country_lookup', - queryString: sane` - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}' - ) - ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) - `, - }), - new WorkbenchQueryPart({ - id: 'ccc', - queryName: 'x', - queryString: sane` - SELECT - os, - CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", - COUNT(DISTINCT session) AS "unique_sessions" - FROM kttm_data - LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country - GROUP BY 1, 2 - ORDER BY 3 DESC - LIMIT 10 - `, - }), - ]) - .materializeHelpers() - .getQueryString(), - ).toEqual(sane` - WITH - "kttm_data" AS ( - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}' - ) - ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) - ), - "country_lookup" AS ( - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}' - ) - ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) - ) - ( - SELECT - os, - CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", - COUNT(DISTINCT session) AS "unique_sessions" - FROM kttm_data - LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country - GROUP BY 1, 2 - ORDER BY 3 DESC - LIMIT 10 - ) - `); - }); - }); - describe('#getIssue', () => { it('works', () => { expect( diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index bee9f07e038..874b10165ab 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -20,7 +20,6 @@ import type { SqlClusteredByClause, SqlExpression, SqlPartitionedByClause, - SqlQuery, } from '@druid-toolkit/query'; import { C, @@ -28,17 +27,18 @@ import { SqlLiteral, SqlOrderByClause, SqlOrderByExpression, - SqlTable, + SqlQuery, } from '@druid-toolkit/query'; import Hjson from 'hjson'; import * as JSONBig from 'json-bigint-native'; import { v4 as uuidv4 } from 'uuid'; -import type { ColumnMetadata, RowColumn } from '../../utils'; -import { deleteKeys, generate8HexId } from '../../utils'; +import type { RowColumn } from '../../utils'; +import { deleteKeys } from '../../utils'; import type { DruidEngine } from '../druid-engine/druid-engine'; import { validDruidEngine } from '../druid-engine/druid-engine'; import type { LastExecution } from '../execution/execution'; +import { validateLastExecution } from '../execution/execution'; import type { ExternalConfig } from '../external-config/external-config'; import { externalConfigToIngestQueryPattern, @@ -46,7 +46,7 @@ import { } from '../ingest-query-pattern/ingest-query-pattern'; import type { QueryContext } from '../query-context/query-context'; -import { WorkbenchQueryPart } from './workbench-query-part'; +const ISSUE_MARKER = '--:ISSUE:'; export interface TabEntry { id: string; @@ -64,10 +64,15 @@ interface IngestionLines { // ----------------------------- export interface WorkbenchQueryValue { - queryParts: WorkbenchQueryPart[]; + queryString: string; queryContext: QueryContext; engine?: DruidEngine; + lastExecution?: LastExecution; unlimited?: boolean; + prefixLines?: number; + + // Legacy + queryParts?: any[]; } export class WorkbenchQuery { @@ -75,8 +80,8 @@ export class WorkbenchQuery { static blank(): WorkbenchQuery { return new WorkbenchQuery({ + queryString: '', queryContext: {}, - queryParts: [WorkbenchQueryPart.blank()], }); } @@ -87,19 +92,15 @@ export class WorkbenchQuery { partitionedByHint: string | undefined, ): WorkbenchQuery { return new WorkbenchQuery({ - queryContext: {}, - queryParts: [ - WorkbenchQueryPart.fromQueryString( - ingestQueryPatternToQuery( - externalConfigToIngestQueryPattern( - externalConfig, - isArrays, - timeExpression, - partitionedByHint, - ), - ).toString(), + queryString: ingestQueryPatternToQuery( + externalConfigToIngestQueryPattern( + externalConfig, + isArrays, + timeExpression, + partitionedByHint, ), - ], + ).toString(), + queryContext: {}, }); } @@ -118,39 +119,21 @@ export class WorkbenchQuery { } } - const queryParts: WorkbenchQueryPart[] = []; + let queryString = ''; let queryContext: QueryContext = {}; for (let i = 0; i < headers.length; i++) { const header = headers[i]; const body = bodies[i]; if (header === 'Context') { queryContext = JSONBig.parse(body); - } else if (header.startsWith('Helper:')) { - queryParts.push( - new WorkbenchQueryPart({ - id: generate8HexId(), - queryName: header.replace(/^Helper:/, '').trim(), - queryString: body, - collapsed: true, - }), - ); } else { - queryParts.push( - new WorkbenchQueryPart({ - id: generate8HexId(), - queryString: body, - }), - ); + queryString = body; } } - if (!queryParts.length) { - queryParts.push(WorkbenchQueryPart.blank()); - } - return new WorkbenchQuery({ + queryString, queryContext, - queryParts, }); } @@ -229,20 +212,44 @@ export class WorkbenchQuery { return { row: Number(m[1]) - 1, column: Number(m[2]) - 1 }; } - public readonly queryParts: WorkbenchQueryPart[]; + static isTaskEngineNeeded(queryString: string): boolean { + return /EXTERN\s*\(|(?:INSERT|REPLACE)\s+INTO/im.test(queryString); + } + + static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined { + // Assuming the queryFragment is no parsable find the prefix that look like: + // REPLACEINTOSELECT + const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index; + if (typeof matchInsertReplaceIndex !== 'number') return; + + const queryStartingWithInsertOrReplace = queryFragment.substring(matchInsertReplaceIndex); + + const matchEnd = queryStartingWithInsertOrReplace.match(/\b(?:SELECT|WITH)\b|$/i); + const fragmentQuery = SqlQuery.maybeParse( + queryStartingWithInsertOrReplace.substring(0, matchEnd?.index) + ' SELECT * FROM t', + ); + if (!fragmentQuery) return; + + return fragmentQuery.getIngestTable()?.getName(); + } + + public readonly queryString: string; public readonly queryContext: QueryContext; public readonly engine?: DruidEngine; + public readonly lastExecution?: LastExecution; public readonly unlimited?: boolean; + public readonly prefixLines?: number; + + public readonly parsedQuery?: SqlQuery; constructor(value: WorkbenchQueryValue) { - let queryParts = value.queryParts; - if (!Array.isArray(queryParts) || !queryParts.length) { - queryParts = [WorkbenchQueryPart.blank()]; + let queryString = value.queryString; + // Back compat to read legacy workbench query + if (typeof queryString === 'undefined' && Array.isArray(value.queryParts)) { + const lastQueryPart = value.queryParts[value.queryParts.length - 1]; + queryString = lastQueryPart.queryString || ''; } - if (!(queryParts instanceof WorkbenchQueryPart)) { - queryParts = queryParts.map(p => new WorkbenchQueryPart(p)); - } - this.queryParts = queryParts; + this.queryString = queryString; this.queryContext = value.queryContext; // Start back compat code for the engine names that might be coming from local storage @@ -255,13 +262,17 @@ export class WorkbenchQuery { // End bac compat code this.engine = validDruidEngine(possibleEngine) ? possibleEngine : undefined; + this.lastExecution = validateLastExecution(value.lastExecution); if (value.unlimited) this.unlimited = true; + this.prefixLines = value.prefixLines; + + this.parsedQuery = SqlQuery.maybeParse(this.queryString); } public valueOf(): WorkbenchQueryValue { return { - queryParts: this.queryParts, + queryString: this.queryString, queryContext: this.queryContext, engine: this.engine, unlimited: this.unlimited, @@ -269,21 +280,17 @@ export class WorkbenchQuery { } public toString(): string { - const { queryParts, queryContext } = this; - return queryParts - .slice(0, queryParts.length - 1) - .flatMap(part => [`===== Helper: ${part.queryName} =====`, part.queryString]) - .concat([ - `===== Query =====`, - this.getLastPart().queryString, - `===== Context =====`, - JSONBig.stringify(queryContext, undefined, 2), - ]) - .join('\n\n'); + const { queryString, queryContext } = this; + return [ + `===== Query =====`, + queryString, + `===== Context =====`, + JSONBig.stringify(queryContext, undefined, 2), + ].join('\n\n'); } - public changeQueryParts(queryParts: WorkbenchQueryPart[]): WorkbenchQuery { - return new WorkbenchQuery({ ...this.valueOf(), queryParts }); + public changeQueryString(queryString: string): WorkbenchQuery { + return new WorkbenchQuery({ ...this.valueOf(), queryString }); } public changeQueryContext(queryContext: QueryContext): WorkbenchQuery { @@ -294,20 +301,28 @@ export class WorkbenchQuery { return new WorkbenchQuery({ ...this.valueOf(), engine }); } + public changeLastExecution(lastExecution: LastExecution | undefined): WorkbenchQuery { + return new WorkbenchQuery({ ...this.valueOf(), lastExecution }); + } + public changeUnlimited(unlimited: boolean): WorkbenchQuery { return new WorkbenchQuery({ ...this.valueOf(), unlimited }); } + public changePrefixLines(prefixLines: number): WorkbenchQuery { + return new WorkbenchQuery({ ...this.valueOf(), prefixLines }); + } + public isTaskEngineNeeded(): boolean { - return this.queryParts.some(part => part.isTaskEngineNeeded()); + return WorkbenchQuery.isTaskEngineNeeded(this.queryString); } public getEffectiveEngine(): DruidEngine { const { engine } = this; if (engine) return engine; const enabledEngines = WorkbenchQuery.getQueryEngines(); - if (this.getLastPart().isJsonLike()) { - if (this.getLastPart().isSqlInJson()) { + if (this.isJsonLike()) { + if (this.isSqlInJson()) { if (enabledEngines.includes('sql-native')) return 'sql-native'; } else { if (enabledEngines.includes('native')) return 'native'; @@ -318,61 +333,60 @@ export class WorkbenchQuery { return enabledEngines[0] || 'sql-native'; } - private getLastPart(): WorkbenchQueryPart { - const { queryParts } = this; - return queryParts[queryParts.length - 1]; - } - - public getId(): string { - return this.getLastPart().id; - } - - public getIds(): string[] { - return this.queryParts.map(queryPart => queryPart.id); - } - - public getQueryName(): string { - return this.getLastPart().queryName || ''; - } - public getQueryString(): string { - return this.getLastPart().queryString; - } - - public getCollapsed(): boolean { - return this.getLastPart().collapsed; + return this.queryString; } public getLastExecution(): LastExecution | undefined { - return this.getLastPart().lastExecution; + return this.lastExecution; } public getParsedQuery(): SqlQuery | undefined { - return this.getLastPart().parsedQuery; + return this.parsedQuery; } public isEmptyQuery(): boolean { - return this.getLastPart().isEmptyQuery(); + return this.queryString.trim() === ''; } public getIssue(): string | undefined { - const lastPart = this.getLastPart(); - if (lastPart.isJsonLike()) { - return lastPart.issueWithJson(); + if (this.isJsonLike()) { + return this.issueWithJson(); } return; } + public isJsonLike(): boolean { + return this.queryString.trim().startsWith('{'); + } + + public issueWithJson(): string | undefined { + try { + Hjson.parse(this.queryString); + } catch (e) { + return e.message; + } + return; + } + + public isSqlInJson(): boolean { + try { + const query = Hjson.parse(this.queryString); + return typeof query.query === 'string'; + } catch { + return false; + } + } + public canPrettify(): boolean { - const lastPart = this.getLastPart(); - return lastPart.isJsonLike(); + return this.isJsonLike(); } public prettify(): WorkbenchQuery { - const lastPart = this.getLastPart(); + const queryString = this.getQueryString(); let parsed; try { - parsed = Hjson.parse(lastPart.queryString); + parsed = Hjson.parse(queryString); } catch { return this; } @@ -381,96 +395,31 @@ export class WorkbenchQuery { public getIngestDatasource(): string | undefined { if (this.getEffectiveEngine() !== 'sql-msq-task') return; - return this.getLastPart().getIngestDatasource(); + + const { queryString, parsedQuery } = this; + if (parsedQuery) { + return parsedQuery.getIngestTable()?.getName(); + } + + if (this.isJsonLike()) return; + + return WorkbenchQuery.getIngestDatasourceFromQueryFragment(queryString); } public isIngestQuery(): boolean { return Boolean(this.getIngestDatasource()); } - private changeLastQueryPart(lastQueryPart: WorkbenchQueryPart): WorkbenchQuery { - const { queryParts } = this; - return this.changeQueryParts(queryParts.slice(0, queryParts.length - 1).concat(lastQueryPart)); - } - - public changeQueryName(queryName: string): WorkbenchQuery { - return this.changeLastQueryPart(this.getLastPart().changeQueryName(queryName)); - } - - public changeQueryString(queryString: string): WorkbenchQuery { - return this.changeLastQueryPart(this.getLastPart().changeQueryString(queryString)); - } - - public changeCollapsed(collapsed: boolean): WorkbenchQuery { - return this.changeLastQueryPart(this.getLastPart().changeCollapsed(collapsed)); - } - - public changeLastExecution(lastExecution: LastExecution | undefined): WorkbenchQuery { - return this.changeLastQueryPart(this.getLastPart().changeLastExecution(lastExecution)); - } - - public clear(): WorkbenchQuery { - return new WorkbenchQuery({ - queryParts: [], - queryContext: {}, - }); - } - public toggleUnlimited(): WorkbenchQuery { const { unlimited } = this; return this.changeUnlimited(!unlimited); } - public hasHelperQueries(): boolean { - return this.queryParts.length > 1; - } - - public materializeHelpers(): WorkbenchQuery { - if (!this.hasHelperQueries()) return this; - const { query } = this.getApiQuery(); - const queryString = query.query; - if (typeof queryString !== 'string') return this; - const lastPart = this.getLastPart(); - return this.changeQueryParts([ - new WorkbenchQueryPart({ - id: lastPart.id, - queryName: lastPart.queryName, - queryString, - }), - ]); - } - - public extractCteHelpers(): WorkbenchQuery { - const { queryParts } = this; - - let changed = false; - const newParts = queryParts.flatMap(queryPart => { - const helpers = queryPart.extractCteHelpers(); - if (helpers) changed = true; - return helpers || [queryPart]; - }); - return changed ? this.changeQueryParts(newParts) : this; - } - public makePreview(): WorkbenchQuery { if (!this.isIngestQuery()) return this; let ret: WorkbenchQuery = this; - // Limit all the helper queries - const parsedQuery = this.getParsedQuery(); - if (parsedQuery) { - const fromExpression = parsedQuery.getFirstFromExpression(); - if (fromExpression instanceof SqlTable) { - const firstTable = fromExpression.getName(); - ret = ret.changeQueryParts( - this.queryParts.map(queryPart => - queryPart.queryName === firstTable ? queryPart.addPreviewLimit() : queryPart, - ), - ); - } - } - // Explicitly select MSQ, adjust the context, set maxNumTasks to the lowest possible and add in ingest mode flags ret = ret.changeEngine('sql-msq-task').changeQueryContext({ ...this.queryContext, @@ -480,8 +429,8 @@ export class WorkbenchQuery { }); // Remove everything pertaining to INSERT INTO / REPLACE INTO from the query string - const newQueryString = parsedQuery - ? parsedQuery + const newQueryString = this.parsedQuery + ? this.parsedQuery .changeInsertClause(undefined) .changeReplaceClause(undefined) .changePartitionedByClause(undefined) @@ -502,18 +451,16 @@ export class WorkbenchQuery { public getApiQuery(makeQueryId: () => string = uuidv4): { engine: DruidEngine; query: Record; - sqlPrefixLines?: number; + prefixLines: number; cancelQueryId?: string; } { - const { queryParts, queryContext, unlimited } = this; - if (!queryParts.length) throw new Error(`should not get here`); + const { queryString, queryContext, unlimited, prefixLines } = this; const engine = this.getEffectiveEngine(); - const lastQueryPart = this.getLastPart(); if (engine === 'native') { let query: any; try { - query = Hjson.parse(lastQueryPart.queryString); + query = Hjson.parse(queryString); } catch (e) { throw new Error( `You have selected the 'native' engine but the query you entered could not be parsed as JSON: ${e.message}`, @@ -530,24 +477,21 @@ export class WorkbenchQuery { return { engine, query, + prefixLines: prefixLines || 0, cancelQueryId, }; } - const prefixParts = queryParts - .slice(0, queryParts.length - 1) - .filter(part => !part.getIngestDatasource()); - let apiQuery: Record = {}; - if (lastQueryPart.isJsonLike()) { + if (this.isJsonLike()) { try { - apiQuery = Hjson.parse(lastQueryPart.queryString); + apiQuery = Hjson.parse(queryString); } catch (e) { throw new Error(`The query you entered could not be parsed as JSON: ${e.message}`); } } else { apiQuery = { - query: lastQueryPart.queryString, + query: queryString, resultFormat: 'array', header: true, typesHeader: true, @@ -555,42 +499,13 @@ export class WorkbenchQuery { }; } - let queryPrepend = ''; - let queryAppend = ''; - - if (prefixParts.length) { - const { insertReplaceLine, overwriteLine, partitionedByLine, clusteredByLine } = - WorkbenchQuery.getIngestionLines(apiQuery.query); - if (insertReplaceLine) { - queryPrepend += insertReplaceLine + '\n'; - if (overwriteLine) { - queryPrepend += overwriteLine + '\n'; - } - - apiQuery.query = WorkbenchQuery.commentOutIngestParts(apiQuery.query); - - if (clusteredByLine) { - queryAppend = '\n' + clusteredByLine + queryAppend; - } - if (partitionedByLine) { - queryAppend = '\n' + partitionedByLine + queryAppend; - } - } - - queryPrepend += 'WITH\n' + prefixParts.map(p => p.toWithPart()).join(',\n') + '\n(\n'; - queryAppend = '\n)' + queryAppend; - } - - let prefixLines = 0; - if (queryPrepend) { - prefixLines = queryPrepend.split('\n').length - 1; - apiQuery.query = queryPrepend + apiQuery.query + queryAppend; - } - - const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query); - if (m) { + const issueIndex = String(apiQuery.query).indexOf(ISSUE_MARKER); + if (issueIndex !== -1) { + const issueComment = String(apiQuery.query) + .slice(issueIndex + ISSUE_MARKER.length) + .split('\n')[0]; throw new Error( - `This query contains an ISSUE comment: ${m[1] + `This query contains an ISSUE comment: ${issueComment .trim() .replace( /\.$/, @@ -628,55 +543,8 @@ export class WorkbenchQuery { return { engine, query: apiQuery, - sqlPrefixLines: prefixLines, + prefixLines: prefixLines || 0, cancelQueryId, }; } - - public getInlineMetadata(): ColumnMetadata[] { - const { queryParts } = this; - if (!queryParts.length) return []; - return queryParts.slice(0, queryParts.length - 1).flatMap(p => p.getInlineMetadata()); - } - - public getPrefix(index: number): WorkbenchQuery { - return this.changeQueryParts(this.queryParts.slice(0, index + 1)); - } - - public getPrefixQueries(): WorkbenchQuery[] { - return this.queryParts.slice(0, this.queryParts.length - 1).map((_, i) => this.getPrefix(i)); - } - - public applyUpdate(newQuery: WorkbenchQuery, index: number): WorkbenchQuery { - return newQuery.changeQueryParts(newQuery.queryParts.concat(this.queryParts.slice(index + 1))); - } - - public duplicate(): WorkbenchQuery { - return this.changeQueryParts(this.queryParts.map(part => part.duplicate())); - } - - public duplicateLast(): WorkbenchQuery { - const { queryParts } = this; - const last = this.getLastPart(); - return this.changeQueryParts(queryParts.concat(last.duplicate())); - } - - public addBlank(): WorkbenchQuery { - const { queryParts } = this; - const last = this.getLastPart(); - return this.changeQueryParts( - queryParts.slice(0, queryParts.length - 1).concat( - last - .changeQueryName(last.queryName || 'q') - .changeCollapsed(true) - .changeLastExecution(undefined), - WorkbenchQueryPart.blank(), - ), - ); - } - - public remove(index: number): WorkbenchQuery { - const { queryParts } = this; - return this.changeQueryParts(queryParts.filter((_, i) => i !== index)); - } } diff --git a/web-console/src/singletons/ace-editor-state-cache.ts b/web-console/src/singletons/ace-editor-state-cache.ts index 0a81173a197..9e1b73aaba8 100644 --- a/web-console/src/singletons/ace-editor-state-cache.ts +++ b/web-console/src/singletons/ace-editor-state-cache.ts @@ -40,9 +40,7 @@ export class AceEditorStateCache { session.setUndoManager(state.undoManager); } - static deleteStates(ids: string[]): void { - for (const id of ids) { - delete AceEditorStateCache.states[id]; - } + static deleteState(id: string): void { + delete AceEditorStateCache.states[id]; } } diff --git a/web-console/src/singletons/execution-state-cache.ts b/web-console/src/singletons/execution-state-cache.ts index 233ed23f7b0..ef1b2d60aeb 100644 --- a/web-console/src/singletons/execution-state-cache.ts +++ b/web-console/src/singletons/execution-state-cache.ts @@ -33,10 +33,4 @@ export class ExecutionStateCache { static deleteState(id: string): void { delete ExecutionStateCache.cache[id]; } - - static deleteStates(ids: string[]): void { - for (const id of ids) { - delete ExecutionStateCache.cache[id]; - } - } } diff --git a/web-console/src/singletons/workbench-running-promises.ts b/web-console/src/singletons/workbench-running-promises.ts index 7f390b2284e..5a14ec136b7 100644 --- a/web-console/src/singletons/workbench-running-promises.ts +++ b/web-console/src/singletons/workbench-running-promises.ts @@ -20,7 +20,7 @@ import type { QueryResult } from '@druid-toolkit/query'; export interface WorkbenchRunningPromise { promise: Promise; - sqlPrefixLines: number | undefined; + prefixLines: number; } export class WorkbenchRunningPromises { @@ -41,10 +41,4 @@ export class WorkbenchRunningPromises { static deletePromise(id: string): void { delete WorkbenchRunningPromises.promises[id]; } - - static deletePromises(ids: string[]): void { - for (const id of ids) { - delete WorkbenchRunningPromises.promises[id]; - } - } } diff --git a/web-console/src/utils/druid-query.spec.ts b/web-console/src/utils/druid-query.spec.ts index 3fd41d6d90f..ee867ff47ea 100644 --- a/web-console/src/utils/druid-query.spec.ts +++ b/web-console/src/utils/druid-query.spec.ts @@ -21,10 +21,10 @@ import { sane } from '@druid-toolkit/query'; import { DruidError, getDruidErrorMessage } from './druid-query'; describe('DruidQuery', () => { - describe('DruidError.parsePosition', () => { + describe('DruidError.extractStartRowColumn', () => { it('works for single error 1', () => { expect( - DruidError.extractPosition({ + DruidError.extractStartRowColumn({ sourceType: 'sql', line: '2', column: '12', @@ -39,7 +39,7 @@ describe('DruidQuery', () => { it('works for range', () => { expect( - DruidError.extractPosition({ + DruidError.extractStartRowColumn({ sourceType: 'sql', line: '1', column: '16', @@ -51,8 +51,37 @@ describe('DruidQuery', () => { ).toEqual({ row: 0, column: 15, - endRow: 0, - endColumn: 16, + }); + }); + }); + + describe('DruidError.extractEndRowColumn', () => { + it('works for single error 1', () => { + expect( + DruidError.extractEndRowColumn({ + sourceType: 'sql', + line: '2', + column: '12', + token: "AS \\'l\\'", + expected: '...', + }), + ).toBeUndefined(); + }); + + it('works for range', () => { + expect( + DruidError.extractEndRowColumn({ + sourceType: 'sql', + line: '1', + column: '16', + endLine: '1', + endColumn: '17', + token: "AS \\'l\\'", + expected: '...', + }), + ).toEqual({ + row: 0, + column: 16, }); }); }); diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts index f87d18feed8..6040950d110 100644 --- a/web-console/src/utils/druid-query.ts +++ b/web-console/src/utils/druid-query.ts @@ -22,8 +22,8 @@ import axios from 'axios'; import { Api } from '../singletons'; +import type { RowColumn } from './general'; import { assemble } from './general'; -import type { RowColumn } from './query-cursor'; const CANCELED_MESSAGE = 'Query canceled by user.'; @@ -109,20 +109,28 @@ export function getDruidErrorMessage(e: any): string { } export class DruidError extends Error { - static extractPosition(context: Record | undefined): RowColumn | undefined { + static extractStartRowColumn( + context: Record | undefined, + offsetLines = 0, + ): RowColumn | undefined { if (context?.sourceType !== 'sql' || !context.line || !context.column) return; - const rowColumn: RowColumn = { - row: Number(context.line) - 1, + return { + row: Number(context.line) - 1 + offsetLines, column: Number(context.column) - 1, }; + } - if (context.endLine && context.endColumn) { - rowColumn.endRow = Number(context.endLine) - 1; - rowColumn.endColumn = Number(context.endColumn) - 1; - } + static extractEndRowColumn( + context: Record | undefined, + offsetLines = 0, + ): RowColumn | undefined { + if (context?.sourceType !== 'sql' || !context.endLine || !context.endColumn) return; - return rowColumn; + return { + row: Number(context.endLine) - 1 + offsetLines, + column: Number(context.endColumn) - 1, + }; } static positionToIndex(str: string, line: number, column: number): number { @@ -256,7 +264,8 @@ export class DruidError extends Error { public errorMessage?: string; public errorMessageWithoutExpectation?: string; public expectation?: string; - public position?: RowColumn; + public startRowColumn?: RowColumn; + public endRowColumn?: RowColumn; public suggestion?: QuerySuggestion; // Deprecated @@ -264,7 +273,7 @@ export class DruidError extends Error { public errorClass?: string; public host?: string; - constructor(e: any, skipLines = 0) { + constructor(e: any, offsetLines = 0) { super(axios.isCancel(e) ? CANCELED_MESSAGE : getDruidErrorMessage(e)); if (axios.isCancel(e)) { this.canceled = true; @@ -286,14 +295,15 @@ export class DruidError extends Error { Object.assign(this, druidErrorResponse); if (this.errorMessage) { - if (skipLines) { + if (offsetLines) { this.errorMessage = this.errorMessage.replace( /line \[(\d+)],/g, - (_, c) => `line [${Number(c) - skipLines}],`, + (_, c) => `line [${Number(c) + offsetLines}],`, ); } - this.position = DruidError.extractPosition(this.context); + this.startRowColumn = DruidError.extractStartRowColumn(this.context, offsetLines); + this.endRowColumn = DruidError.extractEndRowColumn(this.context, offsetLines); this.suggestion = DruidError.getSuggestion(this.errorMessage); const expectationIndex = this.errorMessage.indexOf('Was expecting one of'); diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts index 7b380ae7cf3..a044c0dc23b 100644 --- a/web-console/src/utils/general.spec.ts +++ b/web-console/src/utils/general.spec.ts @@ -175,15 +175,24 @@ describe('general', () => { describe('offsetToRowColumn', () => { it('works', () => { - expect(offsetToRowColumn('Hello\nThis is a test\nstring.', -6)).toBeUndefined(); - expect(offsetToRowColumn('Hello\nThis is a test\nstring.', 666)).toBeUndefined(); - expect(offsetToRowColumn('Hello\nThis is a test\nstring.', 3)).toEqual({ - column: 3, + const str = 'Hello\nThis is a test\nstring.'; + expect(offsetToRowColumn(str, -6)).toBeUndefined(); + expect(offsetToRowColumn(str, 666)).toBeUndefined(); + expect(offsetToRowColumn(str, 3)).toEqual({ row: 0, - }); - expect(offsetToRowColumn('Hello\nThis is a test\nstring.', 24)).toEqual({ column: 3, + }); + expect(offsetToRowColumn(str, 5)).toEqual({ + row: 0, + column: 5, + }); + expect(offsetToRowColumn(str, 24)).toEqual({ row: 2, + column: 3, + }); + expect(offsetToRowColumn(str, str.length)).toEqual({ + row: 2, + column: 7, }); }); }); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index a9264a97680..a871731f6b6 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -156,7 +156,7 @@ function identity(x: T): T { export function lookupBy( array: readonly T[], - keyFn: (x: T, index: number) => string = String, + keyFn: (x: T, index: number) => string | number = String, valueFn?: (x: T, index: number) => Q, ): Record { if (!valueFn) valueFn = identity as any; @@ -521,17 +521,19 @@ export function generate8HexId(): string { return (Math.random() * 1e10).toString(16).replace('.', '').slice(0, 8); } -export function offsetToRowColumn( - str: string, - offset: number, -): { row: number; column: number } | undefined { +export interface RowColumn { + row: number; + column: number; +} + +export function offsetToRowColumn(str: string, offset: number): RowColumn | undefined { // Ensure offset is within the string length if (offset < 0 || offset > str.length) return; const lines = str.split('\n'); for (let row = 0; row < lines.length; row++) { const line = lines[row]; - if (offset < line.length) { + if (offset <= line.length) { return { row, column: offset, diff --git a/web-console/src/utils/query-cursor.ts b/web-console/src/utils/query-cursor.ts index 158d880b82f..94bbe179389 100644 --- a/web-console/src/utils/query-cursor.ts +++ b/web-console/src/utils/query-cursor.ts @@ -19,6 +19,8 @@ import type { SqlBase, SqlQuery } from '@druid-toolkit/query'; import { L } from '@druid-toolkit/query'; +import type { RowColumn } from './general'; + export const EMPTY_LITERAL = L(''); const CRAZY_STRING = '$.X.@.X.$'; @@ -36,13 +38,6 @@ export function prettyPrintSql(b: SqlBase): string { .toString(); } -export interface RowColumn { - row: number; - column: number; - endRow?: number; - endColumn?: number; -} - export function findEmptyLiteralPosition(query: SqlQuery): RowColumn | undefined { const subQueryString = query.walk(b => (b === EMPTY_LITERAL ? L(CRAZY_STRING) : b)).toString(); @@ -54,7 +49,7 @@ export function findEmptyLiteralPosition(query: SqlQuery): RowColumn | undefined const row = lines.length - 1; const lastLine = lines[row]; return { - row, + row: row, column: lastLine.length, }; } diff --git a/web-console/src/utils/sql.spec.ts b/web-console/src/utils/sql.spec.ts new file mode 100644 index 00000000000..08bd45370cd --- /dev/null +++ b/web-console/src/utils/sql.spec.ts @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { sane } from '@druid-toolkit/query'; + +import { findAllSqlQueriesInText, findSqlQueryPrefix } from './sql'; + +describe('sql', () => { + describe('getSqlQueryPrefix', () => { + it('works when whole query parses', () => { + expect( + findSqlQueryPrefix(sane` + SELECT * + FROM wikipedia + `), + ).toMatchInlineSnapshot(` + "SELECT * + FROM wikipedia" + `); + }); + + it('works when there are two queries', () => { + expect( + findSqlQueryPrefix(sane` + SELECT * + FROM wikipedia + + SELECT * + FROM w2 + `), + ).toMatchInlineSnapshot(` + "SELECT * + FROM wikipedia" + `); + }); + + it('works when there are extra closing parens', () => { + expect( + findSqlQueryPrefix(sane` + SELECT * + FROM wikipedia)) lololol + `), + ).toMatchInlineSnapshot(` + "SELECT * + FROM wikipedia" + `); + }); + }); + + describe('findAllSqlQueriesInText', () => { + it('works with separate queries', () => { + const text = sane` + SELECT * + FROM wikipedia + + SELECT * + FROM w2 + LIMIT 5 + + SELECT + `; + + const found = findAllSqlQueriesInText(text); + + expect(found).toMatchInlineSnapshot(` + Array [ + Object { + "endOffset": 23, + "endRowColumn": Object { + "column": 14, + "row": 1, + }, + "sql": "SELECT * + FROM wikipedia", + "startOffset": 0, + "startRowColumn": Object { + "column": 0, + "row": 0, + }, + }, + Object { + "endOffset": 49, + "endRowColumn": Object { + "column": 7, + "row": 5, + }, + "sql": "SELECT * + FROM w2 + LIMIT 5", + "startOffset": 25, + "startRowColumn": Object { + "column": 0, + "row": 3, + }, + }, + ] + `); + }); + + it('works with simple query inside', () => { + const text = sane` + SELECT + "channel", + COUNT(*) AS "Count" + FROM (SELECT * FROM "wikipedia") + GROUP BY 1 + ORDER BY 2 DESC + `; + + const found = findAllSqlQueriesInText(text); + + expect(found).toMatchInlineSnapshot(` + Array [ + Object { + "endOffset": 101, + "endRowColumn": Object { + "column": 15, + "row": 5, + }, + "sql": "SELECT + \\"channel\\", + COUNT(*) AS \\"Count\\" + FROM (SELECT * FROM \\"wikipedia\\") + GROUP BY 1 + ORDER BY 2 DESC", + "startOffset": 0, + "startRowColumn": Object { + "column": 0, + "row": 0, + }, + }, + Object { + "endOffset": 73, + "endRowColumn": Object { + "column": 31, + "row": 3, + }, + "sql": "SELECT * FROM \\"wikipedia\\"", + "startOffset": 48, + "startRowColumn": Object { + "column": 6, + "row": 3, + }, + }, + ] + `); + }); + + it('works with CTE query', () => { + const text = sane` + WITH w1 AS ( + SELECT channel, page FROM "wikipedia" + ) + SELECT + page, + COUNT(*) AS "cnt" + FROM w1 + GROUP BY 1 + ORDER BY 2 DESC + `; + + const found = findAllSqlQueriesInText(text); + + expect(found).toMatchInlineSnapshot(` + Array [ + Object { + "endOffset": 124, + "endRowColumn": Object { + "column": 15, + "row": 8, + }, + "sql": "WITH w1 AS ( + SELECT channel, page FROM \\"wikipedia\\" + ) + SELECT + page, + COUNT(*) AS \\"cnt\\" + FROM w1 + GROUP BY 1 + ORDER BY 2 DESC", + "startOffset": 0, + "startRowColumn": Object { + "column": 0, + "row": 0, + }, + }, + Object { + "endOffset": 52, + "endRowColumn": Object { + "column": 39, + "row": 1, + }, + "sql": "SELECT channel, page FROM \\"wikipedia\\"", + "startOffset": 15, + "startRowColumn": Object { + "column": 2, + "row": 1, + }, + }, + Object { + "endOffset": 124, + "endRowColumn": Object { + "column": 15, + "row": 8, + }, + "sql": "SELECT + page, + COUNT(*) AS \\"cnt\\" + FROM w1 + GROUP BY 1 + ORDER BY 2 DESC", + "startOffset": 55, + "startRowColumn": Object { + "column": 0, + "row": 3, + }, + }, + ] + `); + }); + + it('works with replace query', () => { + const text = sane` + REPLACE INTO "wikipedia" OVERWRITE ALL + WITH "ext" AS (SELECT * + FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}' + ) + ) EXTEND ("isRobot" VARCHAR, "channel" VARCHAR, "timestamp" VARCHAR)) + SELECT + TIME_PARSE("timestamp") AS "__time", + "isRobot", + "channel" + FROM "ext" + PARTITIONED BY DAY + `; + + const found = findAllSqlQueriesInText(text); + + expect(found).toMatchInlineSnapshot(` + Array [ + Object { + "endOffset": 363, + "endRowColumn": Object { + "column": 18, + "row": 13, + }, + "sql": "REPLACE INTO \\"wikipedia\\" OVERWRITE ALL + WITH \\"ext\\" AS (SELECT * + FROM TABLE( + EXTERN( + '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}', + '{\\"type\\":\\"json\\"}' + ) + ) EXTEND (\\"isRobot\\" VARCHAR, \\"channel\\" VARCHAR, \\"timestamp\\" VARCHAR)) + SELECT + TIME_PARSE(\\"timestamp\\") AS \\"__time\\", + \\"isRobot\\", + \\"channel\\" + FROM \\"ext\\" + PARTITIONED BY DAY", + "startOffset": 0, + "startRowColumn": Object { + "column": 0, + "row": 0, + }, + }, + Object { + "endOffset": 344, + "endRowColumn": Object { + "column": 10, + "row": 12, + }, + "sql": "WITH \\"ext\\" AS (SELECT * + FROM TABLE( + EXTERN( + '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}', + '{\\"type\\":\\"json\\"}' + ) + ) EXTEND (\\"isRobot\\" VARCHAR, \\"channel\\" VARCHAR, \\"timestamp\\" VARCHAR)) + SELECT + TIME_PARSE(\\"timestamp\\") AS \\"__time\\", + \\"isRobot\\", + \\"channel\\" + FROM \\"ext\\"", + "startOffset": 39, + "startRowColumn": Object { + "column": 0, + "row": 1, + }, + }, + Object { + "endOffset": 261, + "endRowColumn": Object { + "column": 68, + "row": 7, + }, + "sql": "SELECT * + FROM TABLE( + EXTERN( + '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}', + '{\\"type\\":\\"json\\"}' + ) + ) EXTEND (\\"isRobot\\" VARCHAR, \\"channel\\" VARCHAR, \\"timestamp\\" VARCHAR)", + "startOffset": 54, + "startRowColumn": Object { + "column": 15, + "row": 1, + }, + }, + Object { + "endOffset": 344, + "endRowColumn": Object { + "column": 10, + "row": 12, + }, + "sql": "SELECT + TIME_PARSE(\\"timestamp\\") AS \\"__time\\", + \\"isRobot\\", + \\"channel\\" + FROM \\"ext\\"", + "startOffset": 263, + "startRowColumn": Object { + "column": 0, + "row": 8, + }, + }, + ] + `); + }); + }); +}); diff --git a/web-console/src/utils/sql.ts b/web-console/src/utils/sql.ts index 18019561ac2..7404ee31372 100644 --- a/web-console/src/utils/sql.ts +++ b/web-console/src/utils/sql.ts @@ -16,7 +16,17 @@ * limitations under the License. */ -import { SqlColumn, SqlExpression, SqlFunction, SqlLiteral, SqlStar } from '@druid-toolkit/query'; +import { + SqlColumn, + SqlExpression, + SqlFunction, + SqlLiteral, + SqlQuery, + SqlStar, +} from '@druid-toolkit/query'; + +import type { RowColumn } from './general'; +import { offsetToRowColumn } from './general'; export function timeFormatToSql(timeFormat: string): SqlExpression | undefined { switch (timeFormat) { @@ -60,3 +70,72 @@ export function convertToGroupByExpression(ex: SqlExpression): SqlExpression | u return newEx.as((ex.getOutputName() || 'grouped').replace(/^[a-z]+_/i, '')); } + +function extractQueryPrefix(text: string): string { + let q = SqlQuery.parse(text); + + // The parser will parse a SELECT query with a partitionedByClause and clusteredByClause but that is not valid, remove them from the query + if (!q.getIngestTable() && (q.partitionedByClause || q.clusteredByClause)) { + q = q.changePartitionedByClause(undefined).changeClusteredByClause(undefined); + } + + return q.toString().trimEnd(); +} + +export function findSqlQueryPrefix(text: string): string | undefined { + try { + return extractQueryPrefix(text); + } catch (e) { + const startOffset = e.location?.start?.offset; + if (typeof startOffset !== 'number') return; + const prefix = text.slice(0, startOffset); + // Try to trim to where the error came from + try { + return extractQueryPrefix(prefix); + } catch { + // Try to trim out last word + try { + return extractQueryPrefix(prefix.replace(/\s*\w+$/, '')); + } catch { + return; + } + } + } +} + +export interface QuerySlice { + startOffset: number; + startRowColumn: RowColumn; + endOffset: number; + endRowColumn: RowColumn; + sql: string; +} + +export function findAllSqlQueriesInText(text: string): QuerySlice[] { + const found: QuerySlice[] = []; + + let remainingText = text; + let offset = 0; + let m: RegExpExecArray | null = null; + do { + m = /SELECT|WITH|INSERT|REPLACE/i.exec(remainingText); + if (m) { + const sql = findSqlQueryPrefix(remainingText.slice(m.index)); + const advanceBy = m.index + m[0].length; // Skip the initial word + if (sql) { + const endIndex = m.index + sql.length; + found.push({ + startOffset: offset + m.index, + startRowColumn: offsetToRowColumn(text, offset + m.index)!, + endOffset: offset + endIndex, + endRowColumn: offsetToRowColumn(text, offset + endIndex)!, + sql, + }); + } + remainingText = remainingText.slice(advanceBy); + offset += advanceBy; + } + } while (m); + + return found; +} diff --git a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx index 53ca8b68634..fcae24eb797 100644 --- a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx +++ b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx @@ -164,7 +164,6 @@ export const ColumnEditor = React.memo(function ColumnEditor(props: ColumnEditor ({ query: previewQueryString, processQuery: async (previewQueryString, cancelToken) => { - const taskEngine = WorkbenchQueryPart.isTaskEngineNeeded(previewQueryString); - if (taskEngine) { + if (WorkbenchQuery.isTaskEngineNeeded(previewQueryString)) { return extractResult( await submitTaskQuery({ query: previewQueryString, @@ -872,7 +871,6 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) { ))} {effectiveMode === 'sql' && ( ); diff --git a/web-console/src/views/workbench-view/flexible-query-input/__snapshots__/flexible-query-input.spec.tsx.snap b/web-console/src/views/workbench-view/flexible-query-input/__snapshots__/flexible-query-input.spec.tsx.snap index 8a36beabdaf..a21c1eaf5d0 100644 --- a/web-console/src/views/workbench-view/flexible-query-input/__snapshots__/flexible-query-input.spec.tsx.snap +++ b/web-console/src/views/workbench-view/flexible-query-input/__snapshots__/flexible-query-input.spec.tsx.snap @@ -12,7 +12,7 @@ exports[`FlexibleQueryInput matches snapshot 1`] = ` class="flexible-query-input" >

{ it('matches snapshot', () => { const sqlControl = ( - {}} - /> + {}} /> ); const { container } = render(sqlControl); diff --git a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx index 5f1141aaf21..e9eb3688295 100644 --- a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx +++ b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx @@ -16,11 +16,14 @@ * limitations under the License. */ +import { Intent } from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; import { ResizeSensor2 } from '@blueprintjs/popover2'; -import { C, T } from '@druid-toolkit/query'; +import { C, dedupe, T } from '@druid-toolkit/query'; import type { Ace } from 'ace-builds'; import ace from 'ace-builds'; import classNames from 'classnames'; +import debounce from 'lodash.debounce'; import escape from 'lodash.escape'; import React from 'react'; import AceEditor from 'react-ace'; @@ -32,16 +35,16 @@ import { SQL_KEYWORDS, } from '../../../../lib/keywords'; import { SQL_DATA_TYPES, SQL_FUNCTIONS } from '../../../../lib/sql-docs'; +import { AppToaster } from '../../../singletons'; import { AceEditorStateCache } from '../../../singletons/ace-editor-state-cache'; -import type { ColumnMetadata, RowColumn } from '../../../utils'; -import { uniq } from '../../../utils'; +import type { ColumnMetadata, QuerySlice, RowColumn } from '../../../utils'; +import { findAllSqlQueriesInText, findMap, uniq } from '../../../utils'; import './flexible-query-input.scss'; const langTools = ace.require('ace/ext/language_tools'); const V_PADDING = 10; -const SCROLLBAR = 20; const COMPLETER = { insertMatch: (editor: any, data: Ace.Completion) => { @@ -58,8 +61,8 @@ interface ItemDescription { export interface FlexibleQueryInputProps { queryString: string; onQueryStringChange?: (newQueryString: string) => void; - autoHeight: boolean; - minRows?: number; + runQuerySlice?: (querySlice: QuerySlice) => void; + running?: boolean; showGutter?: boolean; placeholder?: string; columnMetadata?: readonly ColumnMetadata[]; @@ -85,6 +88,8 @@ export class FlexibleQueryInput extends React.PureComponent< FlexibleQueryInputState > { private aceEditor: Ace.Editor | undefined; + private lastFoundQueries: QuerySlice[] = []; + private highlightFoundQuery: { row: number; marker: number } | undefined; static replaceDefaultAutoCompleter(): void { if (!langTools) return; @@ -260,6 +265,14 @@ export class FlexibleQueryInput extends React.PureComponent< }, }); } + + this.markQueries(); + } + + componentDidUpdate(prevProps: Readonly) { + if (this.props.queryString !== prevProps.queryString) { + this.markQueriesDebounced(); + } } componentWillUnmount() { @@ -267,8 +280,42 @@ export class FlexibleQueryInput extends React.PureComponent< if (editorStateId && this.aceEditor) { AceEditorStateCache.saveState(editorStateId, this.aceEditor); } + delete this.aceEditor; } + private findAllQueriesByLine() { + const { queryString } = this.props; + const found = dedupe(findAllSqlQueriesInText(queryString), ({ startRowColumn }) => + String(startRowColumn.row), + ); + if (found.length <= 1) return []; // Do not highlight a single query or no queries + + // Do not report the first query if it is basically the main query minus whitespace + const firstQuery = found[0].sql; + if (firstQuery === queryString.trim()) return found.slice(1); + + return found; + } + + private readonly markQueries = () => { + if (!this.props.runQuerySlice) return; + const { aceEditor } = this; + if (!aceEditor) return; + const session = aceEditor.getSession(); + this.lastFoundQueries = this.findAllQueriesByLine(); + + session.clearBreakpoints(); + this.lastFoundQueries.forEach(({ startRowColumn }) => { + // session.addGutterDecoration(startRowColumn.row, `sub-query-gutter-marker query-${i}`); + session.setBreakpoint( + startRowColumn.row, + `sub-query-gutter-marker query-${startRowColumn.row}`, + ); + }); + }; + + private readonly markQueriesDebounced = debounce(this.markQueries, 900, { trailing: true }); + private readonly handleAceContainerResize = (entries: ResizeObserverEntry[]) => { if (entries.length !== 1) return; this.setState({ editorHeight: entries[0].contentRect.height }); @@ -285,35 +332,16 @@ export class FlexibleQueryInput extends React.PureComponent< if (!aceEditor) return; aceEditor.focus(); // Grab the focus aceEditor.getSelection().moveCursorTo(rowColumn.row, rowColumn.column); - if (rowColumn.endRow && rowColumn.endColumn) { - aceEditor - .getSelection() - .selectToPosition({ row: rowColumn.endRow, column: rowColumn.endColumn }); - } + // If we had an end we could also do + // aceEditor.getSelection().selectToPosition({ row: endRow, column: endColumn }); } renderAce() { - const { - queryString, - onQueryStringChange, - autoHeight, - minRows, - showGutter, - placeholder, - editorStateId, - } = this.props; + const { queryString, onQueryStringChange, showGutter, placeholder, editorStateId } = this.props; const { editorHeight } = this.state; const jsonMode = queryString.trim().startsWith('{'); - let height: number; - if (autoHeight) { - height = - Math.max(queryString.split('\n').length, minRows ?? 2) * 18 + 2 * V_PADDING + SCROLLBAR; - } else { - height = editorHeight; - } - return ( - {autoHeight ? ( - this.renderAce() - ) : ( - -
{this.renderAce()}
- - )} + +
{ + if (!runQuerySlice) return; + const classes = [...(e.target as any).classList]; + if (!classes.includes('sub-query-gutter-marker')) return; + const row = findMap(classes, c => { + const m = /^query-(\d+)$/.exec(c); + return m ? Number(m[1]) : undefined; + }); + if (typeof row === 'undefined') return; + + // Gutter query marker clicked on line ${row} + const slice = this.lastFoundQueries.find( + ({ startRowColumn }) => startRowColumn.row === row, + ); + if (!slice) return; + + if (running) { + AppToaster.show({ + icon: IconNames.WARNING_SIGN, + intent: Intent.WARNING, + message: `Another query is currently running`, + }); + return; + } + + runQuerySlice(slice); + }} + onMouseOver={e => { + if (!runQuerySlice) return; + const aceEditor = this.aceEditor; + if (!aceEditor) return; + + const classes = [...(e.target as any).classList]; + if (!classes.includes('sub-query-gutter-marker')) return; + const row = findMap(classes, c => { + const m = /^query-(\d+)$/.exec(c); + return m ? Number(m[1]) : undefined; + }); + if (typeof row === 'undefined' || this.highlightFoundQuery?.row === row) return; + + const slice = this.lastFoundQueries.find( + ({ startRowColumn }) => startRowColumn.row === row, + ); + if (!slice) return; + const marker = aceEditor + .getSession() + .addMarker( + new ace.Range( + slice.startRowColumn.row, + slice.startRowColumn.column, + slice.endRowColumn.row, + slice.endRowColumn.column, + ), + 'sub-query-highlight', + 'text', + ); + this.highlightFoundQuery = { row, marker }; + }} + onMouseOut={() => { + if (!this.highlightFoundQuery) return; + const aceEditor = this.aceEditor; + if (!aceEditor) return; + aceEditor.getSession().removeMarker(this.highlightFoundQuery.marker); + this.highlightFoundQuery = undefined; + }} + > + {this.renderAce()} +
+
); } diff --git a/web-console/src/views/workbench-view/helper-query/helper-query.scss b/web-console/src/views/workbench-view/helper-query/helper-query.scss deleted file mode 100644 index 4233353c718..00000000000 --- a/web-console/src/views/workbench-view/helper-query/helper-query.scss +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -@import '../../../variables'; - -.helper-query { - position: relative; - @include card-like; - - .query-top-bar { - position: relative; - height: 36px; - display: flex; - align-items: center; - gap: 4px; - white-space: nowrap; - - .corner { - position: absolute; - top: 50%; - right: 3px; - transform: translate(0, -50%); - @include card-background; - } - } - - .flexible-query-input { - border-top: 1px solid rgba($dark-gray1, 0.5); - border-bottom: 1px solid rgba($dark-gray1, 0.5); - } - - .query-control-bar { - position: relative; - width: 100%; - height: 30px; - display: flex; - gap: 10px; - align-items: center; - - .execution-timer-panel, - .execution-summary-panel { - position: absolute; - top: 0; - right: 0; - } - } - - .init-pane { - text-align: center; - flex: 1; - border-top: 1px solid rgba($dark-gray1, 0.5); - - p { - position: relative; - top: 38%; - font-size: 15px; - } - } - - .output-pane { - overflow: hidden; - position: relative; - height: 254px; - border-top: 1px solid rgba($dark-gray1, 0.5); - - > * { - position: absolute; - width: 100%; - height: 100%; - } - - .error-container { - position: relative; - - .execution-error-pane { - position: absolute; - top: 5px; - left: 5px; - right: 5px; - height: 150px; - width: auto; - } - } - } -} diff --git a/web-console/src/views/workbench-view/helper-query/helper-query.tsx b/web-console/src/views/workbench-view/helper-query/helper-query.tsx deleted file mode 100644 index 1b0f7a629e5..00000000000 --- a/web-console/src/views/workbench-view/helper-query/helper-query.tsx +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Button, ButtonGroup, InputGroup, Intent, Menu, MenuItem } from '@blueprintjs/core'; -import { IconNames } from '@blueprintjs/icons'; -import { Popover2 } from '@blueprintjs/popover2'; -import type { QueryResult } from '@druid-toolkit/query'; -import { QueryRunner, SqlQuery } from '@druid-toolkit/query'; -import axios from 'axios'; -import type { JSX } from 'react'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { useStore } from 'zustand'; - -import { Loader, QueryErrorPane } from '../../../components'; -import type { DruidEngine, LastExecution, QueryContext } from '../../../druid-models'; -import { - Execution, - fitExternalConfigPattern, - summarizeExternalConfig, - WorkbenchQuery, -} from '../../../druid-models'; -import { - executionBackgroundStatusCheck, - maybeGetClusterCapacity, - reattachTaskExecution, - submitTaskQuery, -} from '../../../helpers'; -import { usePermanentCallback, useQueryManager } from '../../../hooks'; -import { Api, AppToaster } from '../../../singletons'; -import { ExecutionStateCache } from '../../../singletons/execution-state-cache'; -import { WorkbenchHistory } from '../../../singletons/workbench-history'; -import type { WorkbenchRunningPromise } from '../../../singletons/workbench-running-promises'; -import { WorkbenchRunningPromises } from '../../../singletons/workbench-running-promises'; -import type { ColumnMetadata, QueryAction, RowColumn } from '../../../utils'; -import { DruidError, QueryManager } from '../../../utils'; -import { CapacityAlert } from '../capacity-alert/capacity-alert'; -import type { ExecutionDetailsTab } from '../execution-details-pane/execution-details-pane'; -import { ExecutionErrorPane } from '../execution-error-pane/execution-error-pane'; -import { ExecutionProgressPane } from '../execution-progress-pane/execution-progress-pane'; -import { ExecutionStagesPane } from '../execution-stages-pane/execution-stages-pane'; -import { ExecutionSummaryPanel } from '../execution-summary-panel/execution-summary-panel'; -import { ExecutionTimerPanel } from '../execution-timer-panel/execution-timer-panel'; -import { FlexibleQueryInput } from '../flexible-query-input/flexible-query-input'; -import { IngestSuccessPane } from '../ingest-success-pane/ingest-success-pane'; -import { metadataStateStore } from '../metadata-state-store'; -import { ResultTablePane } from '../result-table-pane/result-table-pane'; -import { RunPanel } from '../run-panel/run-panel'; -import { workStateStore } from '../work-state-store'; - -import './helper-query.scss'; - -const queryRunner = new QueryRunner({ - inflateDateStrategy: 'none', -}); - -export interface HelperQueryProps { - query: WorkbenchQuery; - mandatoryQueryContext: QueryContext | undefined; - columnMetadata: readonly ColumnMetadata[] | undefined; - onQueryChange(newQuery: WorkbenchQuery): void; - onQueryTab(newQuery: WorkbenchQuery, tabName?: string): void; - onDelete(): void; - onDetails(id: string, initTab?: ExecutionDetailsTab): void; - queryEngines: DruidEngine[]; - clusterCapacity: number | undefined; - goToTask(taskId: string): void; -} - -export const HelperQuery = React.memo(function HelperQuery(props: HelperQueryProps) { - const { - query, - columnMetadata, - mandatoryQueryContext, - onQueryChange, - onQueryTab, - onDelete, - onDetails, - queryEngines, - clusterCapacity, - goToTask, - } = props; - const [alertElement, setAlertElement] = useState(); - - // Store the cancellation function for natively run queries allowing us to trigger it only when the user explicitly clicks "cancel" (vs changing tab) - const nativeQueryCancelFnRef = useRef<() => void>(); - - const handleQueryStringChange = usePermanentCallback((queryString: string) => { - onQueryChange(query.changeQueryString(queryString)); - }); - - const parsedQuery = query.getParsedQuery(); - const handleQueryAction = usePermanentCallback((queryAction: QueryAction) => { - if (!(parsedQuery instanceof SqlQuery)) return; - onQueryChange(query.changeQueryString(parsedQuery.apply(queryAction).toString())); - - if (shouldAutoRun()) { - setTimeout(() => void handleRun(false), 20); - } - }); - - function shouldAutoRun(): boolean { - if (query.getEffectiveEngine() !== 'sql-native') return false; - const queryDuration = executionState.data?.result?.queryDuration; - return Boolean(queryDuration && queryDuration < 10000); - } - - const queryInputRef = useRef(null); - - const id = query.getId(); - const [executionState, queryManager] = useQueryManager< - WorkbenchQuery | WorkbenchRunningPromise | LastExecution, - Execution, - Execution, - DruidError - >({ - initQuery: ExecutionStateCache.getState(id) - ? undefined - : WorkbenchRunningPromises.getPromise(id) || query.getLastExecution(), - initState: ExecutionStateCache.getState(id), - processQuery: async (q, cancelToken) => { - if (q instanceof WorkbenchQuery) { - ExecutionStateCache.deleteState(id); - - const { engine, query, sqlPrefixLines, cancelQueryId } = q.getApiQuery(); - - switch (engine) { - case 'sql-msq-task': - return await submitTaskQuery({ - query, - prefixLines: sqlPrefixLines, - cancelToken, - preserveOnTermination: true, - onSubmitted: id => { - onQueryChange(props.query.changeLastExecution({ engine, id })); - }, - }); - - case 'native': - case 'sql-native': { - if (cancelQueryId) { - void cancelToken.promise - .then(cancel => { - if (cancel.message === QueryManager.TERMINATION_MESSAGE) return; - return Api.instance.delete( - `/druid/v2${engine === 'sql-native' ? '/sql' : ''}/${Api.encodePath( - cancelQueryId, - )}`, - ); - }) - .catch(() => {}); - } - - onQueryChange(props.query.changeLastExecution(undefined)); - - let result: QueryResult; - try { - const resultPromise = queryRunner.runQuery({ - query, - extraQueryContext: mandatoryQueryContext, - cancelToken: new axios.CancelToken(cancelFn => { - nativeQueryCancelFnRef.current = cancelFn; - }), - }); - WorkbenchRunningPromises.storePromise(id, { promise: resultPromise, sqlPrefixLines }); - - result = await resultPromise; - nativeQueryCancelFnRef.current = undefined; - } catch (e) { - nativeQueryCancelFnRef.current = undefined; - throw new DruidError(e, sqlPrefixLines); - } - - return Execution.fromResult(engine, result); - } - } - } else if (WorkbenchRunningPromises.isWorkbenchRunningPromise(q)) { - let result: QueryResult; - try { - result = await q.promise; - } catch (e) { - WorkbenchRunningPromises.deletePromise(id); - throw new DruidError(e, q.sqlPrefixLines); - } - - WorkbenchRunningPromises.deletePromise(id); - return Execution.fromResult('sql-native', result); - } else { - switch (q.engine) { - case 'sql-msq-task': - return await reattachTaskExecution({ - id: q.id, - cancelToken, - preserveOnTermination: true, - }); - - default: - throw new Error(`Can not reattach on ${q.engine}`); - } - } - }, - backgroundStatusCheck: executionBackgroundStatusCheck, - swallowBackgroundError: Api.isNetworkError, - }); - - useEffect(() => { - if (!executionState.data) return; - ExecutionStateCache.storeState(id, executionState); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [executionState.data, executionState.error]); - - const incrementWorkVersion = useStore( - workStateStore, - useCallback(state => state.increment, []), - ); - useEffect(() => { - incrementWorkVersion(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [executionState.loading, Boolean(executionState.intermediate)]); - - const execution = executionState.data; - - const incrementMetadataVersion = useStore( - metadataStateStore, - useCallback(state => state.increment, []), - ); - useEffect(() => { - if (execution?.isSuccessfulInsert()) { - incrementMetadataVersion(); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [Boolean(execution?.isSuccessfulInsert())]); - - function moveToPosition(position: RowColumn) { - const currentQueryInput = queryInputRef.current; - if (!currentQueryInput) return; - currentQueryInput.goToPosition(position); - } - - const handleRun = usePermanentCallback(async (preview: boolean) => { - const queryIssue = query.getIssue(); - if (queryIssue) { - const position = WorkbenchQuery.getRowColumnFromIssue(queryIssue); - - AppToaster.show({ - icon: IconNames.ERROR, - intent: Intent.DANGER, - timeout: 90000, - message: queryIssue, - action: position - ? { - text: 'Go to issue', - onClick: () => moveToPosition(position), - } - : undefined, - }); - return; - } - - if (query.getEffectiveEngine() !== 'sql-msq-task') { - WorkbenchHistory.addQueryToHistory(query); - queryManager.runQuery(query); - return; - } - - const effectiveQuery = preview - ? query.makePreview() - : query.setMaxNumTasksIfUnset(clusterCapacity); - - const capacityInfo = await maybeGetClusterCapacity(); - - const effectiveMaxNumTasks = effectiveQuery.queryContext.maxNumTasks ?? 2; - if (capacityInfo && capacityInfo.availableTaskSlots < effectiveMaxNumTasks) { - setAlertElement( - { - queryManager.runQuery(effectiveQuery); - }} - onClose={() => { - setAlertElement(undefined); - }} - />, - ); - } else { - queryManager.runQuery(effectiveQuery); - } - }); - - const collapsed = query.getCollapsed(); - const insertDatasource = query.getIngestDatasource(); - - const statsTaskId: string | undefined = execution?.id; - - let extraInfo: string | undefined; - if (collapsed && parsedQuery instanceof SqlQuery) { - try { - extraInfo = summarizeExternalConfig(fitExternalConfigPattern(parsedQuery)); - } catch {} - } - - const onUserCancel = (message?: string) => { - queryManager.cancelCurrent(message); - nativeQueryCancelFnRef.current?.(); - }; - - return ( -
-
-
- {!collapsed && ( - <> - -
- - {executionState.isLoading() && ( - queryManager.cancelCurrent()} - /> - )} - {(execution || executionState.error) && ( - onDetails(statsTaskId!)} - onReset={() => { - queryManager.reset(); - onQueryChange(props.query.changeLastExecution(undefined)); - ExecutionStateCache.deleteState(id); - }} - /> - )} -
- {!executionState.isInit() && ( -
- {execution && - (execution.result ? ( - - ) : execution.isSuccessfulInsert() ? ( - - ) : execution.error ? ( -
- - {execution.stages && ( - onDetails(statsTaskId!, 'error')} - onWarningClick={() => onDetails(statsTaskId!, 'warnings')} - goToTask={goToTask} - /> - )} -
- ) : ( -
Unknown query execution state
- ))} - {executionState.error && ( - { - moveToPosition(position); - }} - queryString={query.getQueryString()} - onQueryStringChange={handleQueryStringChange} - /> - )} - {executionState.isLoading() && - (executionState.intermediate ? ( - - ) : ( - - ))} -
- )} - - )} - {alertElement} -
- ); -}); diff --git a/web-console/src/views/workbench-view/query-tab/query-tab.scss b/web-console/src/views/workbench-view/query-tab/query-tab.scss index 31cf09489c1..eee0375e0ea 100644 --- a/web-console/src/views/workbench-view/query-tab/query-tab.scss +++ b/web-console/src/views/workbench-view/query-tab/query-tab.scss @@ -49,39 +49,20 @@ $vertical-gap: 6px; width: 100%; top: 0; bottom: 30px + $vertical-gap; - overflow: auto; + @include card-like; + overflow: hidden; - .helper-query { - margin-top: $vertical-gap; + .flexible-query-input { + height: 100%; } - .main-query { - position: relative; - @include card-like; - min-height: 100%; - overflow: hidden; - - &.single { - height: 100%; - - .flexible-query-input { - height: 100%; - } - } - - &.multi { - min-height: calc(100% - 18px); - margin-top: $vertical-gap; - } - - .corner { - position: absolute; - top: 0; - right: 0; - @include card-background; - z-index: 1; - padding: 3px; - } + .corner { + position: absolute; + top: 0; + right: 0; + @include card-background; + z-index: 1; + padding: 3px; } } diff --git a/web-console/src/views/workbench-view/query-tab/query-tab.tsx b/web-console/src/views/workbench-view/query-tab/query-tab.tsx index e3d23a7ab6b..8a4129fc67b 100644 --- a/web-console/src/views/workbench-view/query-tab/query-tab.tsx +++ b/web-console/src/views/workbench-view/query-tab/query-tab.tsx @@ -16,13 +16,11 @@ * limitations under the License. */ -import { Button, Code, Intent, Menu, MenuItem } from '@blueprintjs/core'; +import { Code, Intent } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; -import { Popover2 } from '@blueprintjs/popover2'; import type { QueryResult } from '@druid-toolkit/query'; import { QueryRunner, SqlQuery } from '@druid-toolkit/query'; import axios from 'axios'; -import classNames from 'classnames'; import type { JSX } from 'react'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import SplitterLayout from 'react-splitter-layout'; @@ -43,7 +41,7 @@ import { ExecutionStateCache } from '../../../singletons/execution-state-cache'; import { WorkbenchHistory } from '../../../singletons/workbench-history'; import type { WorkbenchRunningPromise } from '../../../singletons/workbench-running-promises'; import { WorkbenchRunningPromises } from '../../../singletons/workbench-running-promises'; -import type { ColumnMetadata, QueryAction, RowColumn } from '../../../utils'; +import type { ColumnMetadata, QueryAction, QuerySlice, RowColumn } from '../../../utils'; import { DruidError, localStorageGet, @@ -59,7 +57,6 @@ import { ExecutionStagesPane } from '../execution-stages-pane/execution-stages-p import { ExecutionSummaryPanel } from '../execution-summary-panel/execution-summary-panel'; import { ExecutionTimerPanel } from '../execution-timer-panel/execution-timer-panel'; import { FlexibleQueryInput } from '../flexible-query-input/flexible-query-input'; -import { HelperQuery } from '../helper-query/helper-query'; import { IngestSuccessPane } from '../ingest-success-pane/ingest-success-pane'; import { metadataStateStore } from '../metadata-state-store'; import { ResultTablePane } from '../result-table-pane/result-table-pane'; @@ -74,6 +71,7 @@ const queryRunner = new QueryRunner({ export interface QueryTabProps { query: WorkbenchQuery; + id: string; mandatoryQueryContext: QueryContext | undefined; columnMetadata: readonly ColumnMetadata[] | undefined; onQueryChange(newQuery: WorkbenchQuery): void; @@ -88,6 +86,7 @@ export interface QueryTabProps { export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { const { query, + id, columnMetadata, mandatoryQueryContext, onQueryChange, @@ -145,7 +144,6 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { const queryInputRef = useRef(null); - const id = query.getId(); const [executionState, queryManager] = useQueryManager< WorkbenchQuery | WorkbenchRunningPromise | LastExecution, Execution, @@ -159,13 +157,13 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { processQuery: async (q, cancelToken) => { if (q instanceof WorkbenchQuery) { ExecutionStateCache.deleteState(id); - const { engine, query, sqlPrefixLines, cancelQueryId } = q.getApiQuery(); + const { engine, query, prefixLines, cancelQueryId } = q.getApiQuery(); switch (engine) { case 'sql-msq-task': return await submitTaskQuery({ query, - prefixLines: sqlPrefixLines, + prefixLines, cancelToken, preserveOnTermination: true, onSubmitted: id => { @@ -199,13 +197,13 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { nativeQueryCancelFnRef.current = cancelFn; }), }); - WorkbenchRunningPromises.storePromise(id, { promise: resultPromise, sqlPrefixLines }); + WorkbenchRunningPromises.storePromise(id, { promise: resultPromise, prefixLines }); result = await resultPromise; nativeQueryCancelFnRef.current = undefined; } catch (e) { nativeQueryCancelFnRef.current = undefined; - throw new DruidError(e, sqlPrefixLines); + throw new DruidError(e, prefixLines); } return Execution.fromResult(engine, result); @@ -217,7 +215,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { result = await q.promise; } catch (e) { WorkbenchRunningPromises.deletePromise(id); - throw new DruidError(e, q.sqlPrefixLines); + throw new DruidError(e, q.prefixLines); } WorkbenchRunningPromises.deletePromise(id); @@ -274,7 +272,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { currentQueryInput.goToPosition(position); } - const handleRun = usePermanentCallback(async (preview: boolean) => { + const handleRun = usePermanentCallback(async (preview: boolean, querySlice?: QuerySlice) => { const queryIssue = query.getIssue(); if (queryIssue) { const position = WorkbenchQuery.getRowColumnFromIssue(queryIssue); @@ -294,15 +292,22 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { return; } - if (query.getEffectiveEngine() !== 'sql-msq-task') { - WorkbenchHistory.addQueryToHistory(query); - queryManager.runQuery(query); + let effectiveQuery = query; + if (querySlice) { + effectiveQuery = effectiveQuery + .changeQueryString(querySlice.sql) + .changePrefixLines(querySlice.startRowColumn.row); + } + + if (effectiveQuery.getEffectiveEngine() !== 'sql-msq-task') { + WorkbenchHistory.addQueryToHistory(effectiveQuery); + queryManager.runQuery(effectiveQuery); return; } - const effectiveQuery = preview - ? query.makePreview() - : query.setMaxNumTasksIfUnset(clusterCapacity); + effectiveQuery = preview + ? effectiveQuery.makePreview() + : effectiveQuery.setMaxNumTasksIfUnset(clusterCapacity); const capacityInfo = await maybeGetClusterCapacity(); @@ -327,9 +332,6 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { const statsTaskId: string | undefined = execution?.id; - const queryPrefixes = query.getPrefixQueries(); - const extractedCtes = query.extractCteHelpers(); - const onUserCancel = (message?: string) => { queryManager.cancelCurrent(message); nativeQueryCancelFnRef.current?.(); @@ -347,81 +349,22 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) { >
- {queryPrefixes.map((queryPrefix, i) => ( - { - onQueryChange(query.applyUpdate(newQuery, i)); - }} - onQueryTab={onQueryTab} - onDelete={() => { - onQueryChange(query.remove(i)); - }} - onDetails={onDetails} - queryEngines={queryEngines} - clusterCapacity={clusterCapacity} - goToTask={goToTask} - /> - ))} -
- -
- - { - onQueryChange(query.addBlank()); - }} - /> - {extractedCtes !== query && ( - onQueryChange(extractedCtes)} - /> - )} - {query.hasHelperQueries() && ( - onQueryChange(query.materializeHelpers())} - /> - )} - onQueryChange(query.duplicateLast())} - /> - - } - > -
-
+ void handleRun(false, slice)} + running={executionState.loading} + columnMetadata={columnMetadata} + editorStateId={id} + />
; queryEngines: DruidEngine[]; @@ -94,7 +94,7 @@ export interface RunPanelProps { } export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { - const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines, clusterCapacity } = + const { query, onQueryChange, onRun, moreMenu, running, small, queryEngines, clusterCapacity } = props; const [editContextDialogOpen, setEditContextDialogOpen] = useState(false); const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false); @@ -201,7 +201,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {