Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Backport #17147 and #17213 to 31 (#17222) 

* Web console: add support for Dart engine (#17147)

* add console support for Dart engine

This reverts commit 6e46edf15dd55e5c51a1a4068e83deba4f22529b.

* feedback fixes

* surface new fields

* prioratize error over results

* better metadata refresh

* feedback fixes

* Web console: misc fixes to the Explore view (#17213)

* make record table able to hide column

* stickyness

* refactor query log

* fix measure drag

* start nested column dialog

* nested expand

* fix filtering on Measures

* use output name

* fix scrolling

* select all / none

* use ARRAY_CONCAT_AGG

* no need to limit if aggregating

* remove magic number

* better search

* update arg list

* add, don't replace
This commit is contained in:
Vadim Ogievetsky 2024-10-02 10:36:31 -07:00 committed by GitHub
parent 491087fbe3
commit 7dd5e755bb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
52 changed files with 1389 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web-console/script/druid
View File

@ -67,6 +67,7 @@ function _build_distribution() {
&& echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-parquet-extensions\", \"druid-deltalake-extensions\"]" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.export.storage.baseDir=/" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.msq.dart.enabled=true" >> conf/druid/auto/_common/common.runtime.properties \
)
}

1
web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
View File

@ -213,6 +213,7 @@ exports[`HeaderBar matches snapshot 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",

49
web-console/src/druid-models/dart/dart-query-entry.mock.ts Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { DartQueryEntry } from './dart-query-entry';
export const DART_QUERIES: DartQueryEntry[] = [
{
sqlQueryId: '77b2344c-0a1f-4aa0-b127-de6fbc0c2b57',
dartQueryId: '99cdba0d-ed77-433d-9adc-0562d816e105',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:21.194Z',
state: 'RUNNING',
},
{
sqlQueryId: '45441cf5-d8b7-46cb-b6d8-682334f056ef',
dartQueryId: '25af9bff-004d-494e-b562-2752dc3779c8',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:22.854Z',
state: 'CANCELED',
},
{
sqlQueryId: 'f7257c78-6bbe-439d-99ba-f4998b300770',
dartQueryId: 'f7c2d644-9c40-4d61-9fdb-7b0e15219886',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:24.425Z',
state: 'ACCEPTED',
},
];

27
web-console/src/druid-models/dart/dart-query-entry.ts Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface DartQueryEntry {
sqlQueryId: string;
dartQueryId: string;
sql: string;
authenticator: string;
identity: string;
startTime: string;
state: 'ACCEPTED' | 'RUNNING' | 'CANCELED';
}

9
web-console/src/druid-models/druid-engine/druid-engine.ts
View File

@ -16,9 +16,14 @@
* limitations under the License.
*/
export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task';
export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task' | 'sql-msq-dart';
export const DRUID_ENGINES: DruidEngine[] = ['native', 'sql-native', 'sql-msq-task'];
export const DRUID_ENGINES: DruidEngine[] = [
'native',
'sql-native',
'sql-msq-task',
'sql-msq-dart',
];
export function validDruidEngine(
possibleDruidEngine: string | undefined,

1
web-console/src/druid-models/index.ts
View File

@ -20,6 +20,7 @@ export * from './async-query/async-query';
export * from './compaction-config/compaction-config';
export * from './compaction-status/compaction-status';
export * from './coordinator-dynamic-config/coordinator-dynamic-config';
export * from './dart/dart-query-entry';
export * from './dimension-spec/dimension-spec';
export * from './druid-engine/druid-engine';
export * from './execution/execution';

23
web-console/src/druid-models/stages/stages.ts
View File

@ -18,6 +18,7 @@
import { max, sum } from 'd3-array';
import { AutoForm } from '../../components';
import { countBy, deleteKeys, filterMap, groupByAsMap, oneOf, zeroDivide } from '../../utils';
import type { InputFormat } from '../input-format/input-format';
import type { InputSource } from '../input-source/input-source';
@ -252,26 +253,16 @@ export const CPUS_COUNTER_FIELDS: CpusCounterFields[] = [
export function cpusCounterFieldTitle(k: CpusCounterFields) {
switch (k) {
case 'main':
return 'Main';
case 'collectKeyStatistics':
return 'Collect key stats';
case 'mergeInput':
return 'Merge input';
case 'hashPartitionOutput':
return 'Hash partition out';
case 'mixOutput':
return 'Mix output';
case 'sortOutput':
return 'Sort output';
default:
return k;
// main
// mergeInput
// hashPartitionOutput
// mixOutput
// sortOutput
return AutoForm.makeLabelName(k);
}
}

6
web-console/src/druid-models/workbench-query/workbench-query.ts
View File

@ -528,7 +528,7 @@ export class WorkbenchQuery {
};
let cancelQueryId: string | undefined;
if (engine === 'sql-native') {
if (engine === 'sql-native' || engine === 'sql-msq-dart') {
cancelQueryId = apiQuery.context.sqlQueryId;
if (!cancelQueryId) {
// If the sqlQueryId is not explicitly set on the context generate one, so it is possible to cancel the query.
@ -550,6 +550,10 @@ export class WorkbenchQuery {
apiQuery.context.sqlStringifyArrays ??= false;
}
if (engine === 'sql-msq-dart') {
apiQuery.context.fullReport ??= true;
}
if (Array.isArray(queryParameters) && queryParameters.length) {
apiQuery.parameters = queryParameters;
}

32
web-console/src/helpers/capabilities.ts
View File

@ -37,6 +37,7 @@ export type QueryType = 'none' | 'nativeOnly' | 'nativeAndSql';
export interface CapabilitiesValue {
queryType: QueryType;
multiStageQueryTask: boolean;
multiStageQueryDart: boolean;
coordinator: boolean;
overlord: boolean;
maxTaskSlots?: number;
@ -53,6 +54,7 @@ export class Capabilities {
private readonly queryType: QueryType;
private readonly multiStageQueryTask: boolean;
private readonly multiStageQueryDart: boolean;
private readonly coordinator: boolean;
private readonly overlord: boolean;
private readonly maxTaskSlots?: number;
@ -139,6 +141,15 @@ export class Capabilities {
}
}
static async detectMultiStageQueryDart(): Promise<boolean> {
try {
const resp = await Api.instance.get(`/druid/v2/sql/dart/enabled?capabilities`);
return Boolean(resp.data.enabled);
} catch {
return false;
}
}
static async detectCapabilities(): Promise<Capabilities | undefined> {
const queryType = await Capabilities.detectQueryType();
if (typeof queryType === 'undefined') return;
@ -154,11 +165,15 @@ export class Capabilities {
coordinator = overlord = await Capabilities.detectManagementProxy();
}
const multiStageQueryTask = await Capabilities.detectMultiStageQueryTask();
const [multiStageQueryTask, multiStageQueryDart] = await Promise.all([
Capabilities.detectMultiStageQueryTask(),
Capabilities.detectMultiStageQueryDart(),
]);
return new Capabilities({
queryType,
multiStageQueryTask,
multiStageQueryDart,
coordinator,
overlord,
});
@ -179,6 +194,7 @@ export class Capabilities {
constructor(value: CapabilitiesValue) {
this.queryType = value.queryType;
this.multiStageQueryTask = value.multiStageQueryTask;
this.multiStageQueryDart = value.multiStageQueryDart;
this.coordinator = value.coordinator;
this.overlord = value.overlord;
this.maxTaskSlots = value.maxTaskSlots;
@ -188,6 +204,7 @@ export class Capabilities {
return {
queryType: this.queryType,
multiStageQueryTask: this.multiStageQueryTask,
multiStageQueryDart: this.multiStageQueryDart,
coordinator: this.coordinator,
overlord: this.overlord,
maxTaskSlots: this.maxTaskSlots,
@ -248,6 +265,10 @@ export class Capabilities {
return this.multiStageQueryTask;
}
public hasMultiStageQueryDart(): boolean {
return this.multiStageQueryDart;
}
public getSupportedQueryEngines(): DruidEngine[] {
const queryEngines: DruidEngine[] = ['native'];
if (this.hasSql()) {
@ -256,6 +277,9 @@ export class Capabilities {
if (this.hasMultiStageQueryTask()) {
queryEngines.push('sql-msq-task');
}
if (this.hasMultiStageQueryDart()) {
queryEngines.push('sql-msq-dart');
}
return queryEngines;
}
@ -282,36 +306,42 @@ export class Capabilities {
Capabilities.FULL = new Capabilities({
queryType: 'nativeAndSql',
multiStageQueryTask: true,
multiStageQueryDart: true,
coordinator: true,
overlord: true,
});
Capabilities.NO_SQL = new Capabilities({
queryType: 'nativeOnly',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR_OVERLORD = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: false,
});
Capabilities.OVERLORD = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: false,
overlord: true,
});
Capabilities.NO_PROXY = new Capabilities({
queryType: 'nativeAndSql',
multiStageQueryTask: true,
multiStageQueryDart: false,
coordinator: false,
overlord: false,
});

13
web-console/src/utils/druid-query.ts
View File

@ -342,6 +342,19 @@ export async function queryDruidSql<T = any>(
return sqlResultResp.data;
}
export async function queryDruidSqlDart<T = any>(
sqlQueryPayload: Record<string, any>,
cancelToken?: CancelToken,
): Promise<T[]> {
let sqlResultResp: AxiosResponse;
try {
sqlResultResp = await Api.instance.post('/druid/v2/sql/dart', sqlQueryPayload, { cancelToken });
} catch (e) {
throw new Error(getDruidErrorMessage(e));
}
return sqlResultResp.data;
}
export interface QueryExplanation {
query: any;
signature: { name: string; type: string }[];

2
web-console/src/utils/local-storage-keys.tsx
View File

@ -53,10 +53,12 @@ export const LocalStorageKeys = {
WORKBENCH_PANE_SIZE: 'workbench-pane-size' as const,
WORKBENCH_HISTORY: 'workbench-history' as const,
WORKBENCH_TASK_PANEL: 'workbench-task-panel' as const,
WORKBENCH_DART_PANEL: 'workbench-dart-panel' as const,
SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const,
EXPLORE_STATE: 'explore-state' as const,
EXPLORE_STICKY: 'explore-sticky' as const,
};
export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys];

1
web-console/src/utils/table-helpers.ts
View File

@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination {
export interface ColumnHint {
displayName?: string;
group?: string;
hidden?: boolean;
expressionForWhere?: SqlExpression;
formatter?: (x: any) => string;
}

22
web-console/src/views/explore-view/components/control-pane/control-pane.tsx
View File

@ -194,7 +194,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
};
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<ExpressionMeta>
allowReordering
values={effectiveValue ? [effectiveValue] : []}
onValuesChange={vs => onValueChange(vs[0])}
@ -223,7 +223,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
);
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<ExpressionMeta>
allowReordering
values={effectiveValue as ExpressionMeta[]}
onValuesChange={onValueChange}
@ -266,7 +266,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
case 'measure': {
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<Measure>
values={effectiveValue ? [effectiveValue] : []}
onValuesChange={vs => onValueChange(vs[0])}
singleton
@ -284,9 +284,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
/>
),
onDropColumn: column => {
const measures = Measure.getPossibleMeasuresForColumn(column);
if (!measures.length) return;
onValueChange(measures[0]);
const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue || effectiveValue.name !== p.name,
);
if (!candidateMeasures.length) return;
onValueChange(candidateMeasures[0]);
},
onDropMeasure: onValueChange,
};
@ -313,11 +315,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
/>
),
onDropColumn: column => {
const measures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue.some((v: ExpressionMeta) => v.name === p.name),
const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue.some((v: Measure) => v.name === p.name),
);
if (!measures.length) return;
onValueChange(effectiveValue.concat(measures[0]));
if (!candidateMeasures.length) return;
onValueChange(effectiveValue.concat(candidateMeasures[0]));
},
onDropMeasure: measure => {
onValueChange(effectiveValue.concat(measure));

1
web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx
View File

@ -61,6 +61,7 @@ export const NamedExpressionsInput = function NamedExpressionsInput<
const onDragOver = useCallback(
(e: React.DragEvent, i: number) => {
if (dragIndex === -1) return;
const targetRect = e.currentTarget.getBoundingClientRect();
const before = e.clientX - targetRect.left <= targetRect.width / 2;
setDropBefore(before);

1
web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx
View File

@ -52,6 +52,7 @@ export const ContainsFilterControl = React.memo(function ContainsFilterControl(
),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
// eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
[querySource.query, filter, column, contains, negated],

1
web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx
View File

@ -58,6 +58,7 @@ export const RegexpFilterControl = React.memo(function RegexpFilterControl(
SqlExpression.and(filter, regexp ? filterPatternToExpression(filterPattern) : undefined),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
// eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
[querySource.query, filter, column, regexp, negated],

92
web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx
View File

@ -16,14 +16,15 @@
* limitations under the License.
*/
import { FormGroup, InputGroup, Menu, MenuItem } from '@blueprintjs/core';
import { FormGroup, Menu, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { QueryResult, SqlQuery, ValuesFilterPattern } from '@druid-toolkit/query';
import { C, F, L, SqlExpression, SqlLiteral } from '@druid-toolkit/query';
import type { QueryResult, ValuesFilterPattern } from '@druid-toolkit/query';
import { C, F, SqlExpression, SqlQuery } from '@druid-toolkit/query';
import React, { useMemo, useState } from 'react';
import { ClearableInput } from '../../../../../../components';
import { useQueryManager } from '../../../../../../hooks';
import { caseInsensitiveContains } from '../../../../../../utils';
import { caseInsensitiveContains, filterMap } from '../../../../../../utils';
import type { QuerySource } from '../../../../models';
import { toggle } from '../../../../utils';
import { ColumnValue } from '../../column-value/column-value';
@ -46,21 +47,21 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
const [initValues] = useState(selectedValues);
const [searchString, setSearchString] = useState('');
const valuesQuery = useMemo(() => {
const columnRef = C(column);
const queryParts: string[] = [`SELECT ${columnRef.as('c')}`, `FROM (${querySource.query})`];
const filterEx = SqlExpression.and(
filter,
searchString ? F('ICONTAINS_STRING', columnRef, L(searchString)) : undefined,
);
if (!(filterEx instanceof SqlLiteral)) {
queryParts.push(`WHERE ${filterEx}`);
}
queryParts.push(`GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 101`);
return queryParts.join('\n');
}, [querySource.query, filter, column, searchString]);
const valuesQuery = useMemo(
() =>
SqlQuery.from(querySource.query)
.addSelect(C(column).as('c'), { addToGroupBy: 'end' })
.changeWhereExpression(
SqlExpression.and(
filter,
searchString ? F('ICONTAINS_STRING', C(column), searchString) : undefined,
),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
[querySource.query, filter, column, searchString],
);
const [valuesState] = useQueryManager<string, any[]>({
query: valuesQuery,
@ -77,42 +78,37 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
if (values) {
valuesToShow = valuesToShow.concat(values.filter(v => !initValues.includes(v)));
}
if (searchString) {
valuesToShow = valuesToShow.filter(v => caseInsensitiveContains(v, searchString));
}
const showSearch = querySource.columns.find(c => c.name === column)?.sqlType !== 'BOOLEAN';
return (
<FormGroup className="values-filter-control">
{showSearch && (
<InputGroup
value={searchString}
onChange={e => setSearchString(e.target.value)}
placeholder="Search"
/>
<ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
)}
<Menu className="value-list">
{valuesToShow.map((v, i) => (
<MenuItem
key={i}
icon={
selectedValues.includes(v)
? negated
? IconNames.DELETE
: IconNames.TICK_CIRCLE
: IconNames.CIRCLE
}
text={<ColumnValue value={v} />}
shouldDismissPopover={false}
onClick={e => {
setFilterPattern({
...filterPattern,
values: e.altKey ? [v] : toggle(selectedValues, v),
});
}}
/>
))}
{filterMap(valuesToShow, (v, i) => {
if (!caseInsensitiveContains(v, searchString)) return;
return (
<MenuItem
key={i}
icon={
selectedValues.includes(v)
? negated
? IconNames.DELETE
: IconNames.TICK_CIRCLE
: IconNames.CIRCLE
}
text={<ColumnValue value={v} />}
shouldDismissPopover={false}
onClick={e => {
setFilterPattern({
...filterPattern,
values: e.altKey ? [v] : toggle(selectedValues, v),
});
}}
/>
);
})}
{valuesState.loading && <MenuItem icon={IconNames.BLANK} text="Loading..." disabled />}
</Menu>
</FormGroup>

4
web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx
View File

@ -428,6 +428,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
columns={columnNester(
queryResult.header.map((column, i) => {
const h = column.name;
const hint = columnHints?.get(h);
const icon = showTypeIcons ? columnToIcon(column) : undefined;
return {
@ -446,9 +447,10 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
},
headerClassName: getHeaderClassName(h),
accessor: String(i),
show: !hint?.hidden,
Cell(row) {
const value = row.value;
const formatter = columnHints?.get(h)?.formatter || formatNumber;
const formatter = hint?.formatter || formatNumber;
return (
<div>
<Popover content={<Deferred content={() => getCellMenu(column, i, value)} />}>

7
web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx
View File

@ -48,9 +48,8 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
if (!expression) return;
return SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
.addSelect(F.cast(expression, 'VARCHAR').as('v'), { addToGroupBy: 'end' })
.applyIf(
querySource.baseColumns.find(column => column.isTimeColumn()),
q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
)
.changeLimitValue(100)
.toString();
@ -151,7 +150,7 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
} else {
onApply(
querySource.changeColumn(initExpressionName, newExpression),
new Map([[initExpression.getOutputName()!, newExpression.getOutputName()!]]),
new Map([[initExpressionName, newExpression.getOutputName()!]]),
);
}
} else {

5
web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx
View File

@ -57,9 +57,8 @@ export const MeasureDialog = React.memo(function MeasureDialog(props: MeasureDia
.changeWithParts([SqlWithPart.simple('t', QuerySource.stripToBaseSource(querySource.query))])
.addSelect(L('Overall').as('label'))
.addSelect(expression.as('value'))
.applyIf(
querySource.baseColumns.find(column => column.isTimeColumn()),
q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
)
.toString();
}, [querySource.query, formula]);

44
web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../../../variables';
.nested-column-dialog {
&.#{$bp-ns}-dialog {
width: 50vw;
min-height: 540px;
}
.#{$bp-ns}-dialog-body {
display: flex;
flex-direction: column;
.path-selector {
flex: 1;
padding: 5px 0;
height: 400px;
overflow: auto;
border-left: 1px solid rgba(15, 19, 32, 0.4);
border-right: 1px solid rgba(15, 19, 32, 0.4);
}
}
.#{$bp-ns}-dialog-footer {
margin-top: 0;
}
}

176
web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Button,
ButtonGroup,
Classes,
Dialog,
FormGroup,
InputGroup,
Intent,
Menu,
Tag,
} from '@blueprintjs/core';
import type { SqlExpression } from '@druid-toolkit/query';
import { type QueryResult, F, sql, SqlFunction, SqlQuery } from '@druid-toolkit/query';
import React, { useState } from 'react';
import { ClearableInput, Loader, MenuCheckbox } from '../../../../../components';
import { useQueryManager } from '../../../../../hooks';
import { caseInsensitiveContains, filterMap, pluralIfNeeded } from '../../../../../utils';
import { ExpressionMeta, QuerySource } from '../../../models';
import { toggle } from '../../../utils';
import './nested-column-dialog.scss';
const ARRAY_CONCAT_AGG_SIZE = 10000;
export interface NestedColumnDialogProps {
nestedColumn: SqlExpression;
onApply(newQuery: SqlQuery): void;
querySource: QuerySource;
runSqlQuery(query: string | SqlQuery): Promise<QueryResult>;
onClose(): void;
}
export const NestedColumnDialog = React.memo(function NestedColumnDialog(
props: NestedColumnDialogProps,
) {
const { nestedColumn, onApply, querySource, runSqlQuery, onClose } = props;
const [searchString, setSearchString] = useState('');
const [selectedPaths, setSelectedPaths] = useState<string[]>([]);
const [namingScheme, setNamingScheme] = useState(`${nestedColumn.getFirstColumnName()}[%]`);
const [pathsState] = useQueryManager({
query: nestedColumn,
processQuery: async nestedColumn => {
const query = SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
.addSelect(
SqlFunction.decorated('ARRAY_CONCAT_AGG', 'DISTINCT', [
F('JSON_PATHS', nestedColumn),
ARRAY_CONCAT_AGG_SIZE,
]),
)
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
);
const pathResult = await runSqlQuery(query);
const paths = pathResult.rows[0]?.[0];
if (!Array.isArray(paths)) throw new Error('Could not get paths');
return paths;
},
});
const paths = pathsState.data;
return (
<Dialog className="nested-column-dialog" isOpen onClose={onClose} title="Expand nested column">
<div className={Classes.DIALOG_BODY}>
<p>
Replace <Tag minimal>{String(nestedColumn.getOutputName())}</Tag> with path expansions for
the selected paths.
</p>
{pathsState.isLoading() && <Loader />}
{pathsState.getErrorMessage()}
{paths && (
<FormGroup>
<ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
<Menu className="path-selector">
{filterMap(paths, (path, i) => {
if (!caseInsensitiveContains(path, searchString)) return;
return (
<MenuCheckbox
key={i}
checked={selectedPaths.includes(path)}
onChange={() => setSelectedPaths(toggle(selectedPaths, path))}
text={path}
/>
);
})}
</Menu>
<ButtonGroup fill>
<Button
text="Select all"
onClick={() =>
// Select all paths matching the shown search string
setSelectedPaths(
paths.filter(path => caseInsensitiveContains(path, searchString)),
)
}
/>
<Button
text="Select none"
onClick={() =>
// Remove from selection all the paths matching the search string
setSelectedPaths(
selectedPaths.filter(path => !caseInsensitiveContains(path, searchString)),
)
}
/>
</ButtonGroup>
</FormGroup>
)}
<FormGroup label="Nameing scheme">
<InputGroup
value={namingScheme}
onChange={e => {
setNamingScheme(e.target.value.slice(0, ExpressionMeta.MAX_NAME_LENGTH));
}}
/>
</FormGroup>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<div className="edit-column-dialog-buttons">
<Button text="Cancel" onClick={onClose} />
<Button
text={
selectedPaths.length
? `Add ${pluralIfNeeded(selectedPaths.length, 'column')}`
: 'Select path'
}
disabled={!selectedPaths.length}
intent={Intent.PRIMARY}
onClick={() => {
const effectiveNamingScheme = namingScheme.includes('%')
? namingScheme
: namingScheme + '[%]';
onApply(
querySource.addColumnAfter(
nestedColumn.getOutputName()!,
...selectedPaths.map(path =>
F('JSON_VALUE', nestedColumn, path).as(
querySource.getAvailableName(
effectiveNamingScheme.replaceAll('%', path.replace(/^\$\./, '')),
),
),
),
),
);
onClose();
}}
/>
</div>
</div>
</div>
</Dialog>
);
});

50
web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx
View File

@ -40,6 +40,7 @@ import type { Rename } from '../../utils';
import { ColumnDialog } from './column-dialog/column-dialog';
import { MeasureDialog } from './measure-dialog/measure-dialog';
import { NestedColumnDialog } from './nested-column-dialog/nested-column-dialog';
import './resource-pane.scss';
@ -67,6 +68,9 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
const [columnSearch, setColumnSearch] = useState('');
const [columnEditorOpenOn, setColumnEditorOpenOn] = useState<ColumnEditorOpenOn | undefined>();
const [nestedColumnEditorOpenOn, setNestedColumnEditorOpenOn] = useState<
SqlExpression | undefined
>();
const [measureEditorOpenOn, setMeasureEditorOpenOn] = useState<MeasureEditorOpenOn | undefined>();
function applyUtil(nameTransform: (columnName: string) => string) {
@ -112,6 +116,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
<div className="column-resource-list">
{filterMap(querySource.columns, (column, i) => {
const columnName = column.name;
const isNestedColumn = column.nativeType === 'COMPLEX<json>';
if (!caseInsensitiveContains(columnName, columnSearch)) return;
return (
<Popover
@ -120,19 +125,33 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
position="right"
content={
<Menu>
{onFilter && (
{isNestedColumn ? (
<MenuItem
icon={IconNames.FILTER}
text="Filter"
onClick={() => onFilter(column)}
icon={IconNames.EXPAND_ALL}
text="Expand nested column"
onClick={() =>
setNestedColumnEditorOpenOn(
querySource.getSourceExpressionForColumn(columnName),
)
}
/>
) : (
<>
{onFilter && (
<MenuItem
icon={IconNames.FILTER}
text="Filter"
onClick={() => onFilter(column)}
/>
)}
<MenuItem
icon={IconNames.EYE_OPEN}
text="Show"
onClick={() => onShowColumn(column)}
/>
<MenuDivider />
</>
)}
<MenuItem
icon={IconNames.EYE_OPEN}
text="Show"
onClick={() => onShowColumn(column)}
/>
<MenuDivider />
<MenuItem
icon={IconNames.EDIT}
text="Edit"
@ -165,7 +184,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
>
<div
className={Classes.MENU_ITEM}
draggable
draggable={!isNestedColumn}
onDragStart={e => {
e.dataTransfer.effectAllowed = 'all';
DragHelper.dragColumn = column;
@ -268,6 +287,15 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
onClose={() => setColumnEditorOpenOn(undefined)}
/>
)}
{nestedColumnEditorOpenOn && (
<NestedColumnDialog
nestedColumn={nestedColumnEditorOpenOn}
onApply={newQuery => onQueryChange(newQuery, undefined)}
querySource={querySource}
runSqlQuery={runSqlQuery}
onClose={() => setNestedColumnEditorOpenOn(undefined)}
/>
)}
{measureEditorOpenOn && (
<MeasureDialog
initMeasure={measureEditorOpenOn.measure}

66
web-console/src/views/explore-view/explore-view.tsx
View File

@ -30,7 +30,15 @@ import { useStore } from 'zustand';
import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
import { useHashAndLocalStorageHybridState, useQueryManager } from '../../hooks';
import { Api, AppToaster } from '../../singletons';
import { DruidError, LocalStorageKeys, queryDruidSql } from '../../utils';
import {
DruidError,
isEmpty,
localStorageGetJson,
LocalStorageKeys,
localStorageSetJson,
mapRecord,
queryDruidSql,
} from '../../utils';
import {
ControlPane,
@ -50,29 +58,14 @@ import { QuerySource } from './models';
import { ModuleRepository } from './module-repository/module-repository';
import { rewriteAggregate, rewriteMaxDataTime } from './query-macros';
import type { Rename } from './utils';
import { adjustTransferValue, normalizeType } from './utils';
import { adjustTransferValue, normalizeType, QueryLog } from './utils';
import './explore-view.scss';
// ---------------------------------------
const QUERY_LOG = new QueryLog();
interface QueryHistoryEntry {
time: Date;
sqlQuery: string;
}
const MAX_PAST_QUERIES = 10;
const QUERY_HISTORY: QueryHistoryEntry[] = [];
function addQueryToHistory(sqlQuery: string): void {
QUERY_HISTORY.unshift({ time: new Date(), sqlQuery });
while (QUERY_HISTORY.length > MAX_PAST_QUERIES) QUERY_HISTORY.pop();
}
function getFormattedQueryHistory(): string {
return QUERY_HISTORY.map(
({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`,
).join('\n\n-----------------------------------------------------\n\n');
function getStickyParameterValuesForModule(moduleId: string): ParameterValues {
return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {};
}
// ---------------------------------------
@ -81,7 +74,7 @@ const queryRunner = new QueryRunner({
inflateDateStrategy: 'fromSqlTypes',
executor: async (sqlQueryPayload, isSql, cancelToken) => {
if (!isSql) throw new Error('should never get here');
addQueryToHistory(sqlQueryPayload.query);
QUERY_LOG.addQuery(sqlQueryPayload.query);
return Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken });
},
});
@ -90,6 +83,9 @@ async function runSqlQuery(query: string | SqlQuery): Promise<QueryResult> {
try {
return await queryRunner.runQuery({
query,
defaultQueryContext: {
sqlStringifyArrays: false,
},
});
} catch (e) {
throw new DruidError(e);
@ -193,10 +189,25 @@ export const ExploreView = React.memo(function ExploreView() {
}
function resetParameterValues() {
setParameterValues({});
setParameterValues(getStickyParameterValuesForModule(moduleId));
}
function updateParameterValues(newParameterValues: ParameterValues) {
// Evaluate sticky-ness
if (module) {
const currentExploreSticky = localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY) || {};
const currentModuleSticky = currentExploreSticky[moduleId] || {};
const newModuleSticky = {
...currentModuleSticky,
...mapRecord(newParameterValues, (v, k) => (module.parameters[k]?.sticky ? v : undefined)),
};
localStorageSetJson(LocalStorageKeys.EXPLORE_STICKY, {
...currentExploreSticky,
[moduleId]: isEmpty(newModuleSticky) ? undefined : newModuleSticky,
});
}
setParameterValues({ ...parameterValues, ...newParameterValues });
}
@ -311,7 +322,8 @@ export const ExploreView = React.memo(function ExploreView() {
]}
selectedModuleId={moduleId}
onSelectedModuleIdChange={newModuleId => {
const newParameterValues: ParameterValues = {};
const newParameterValues = getStickyParameterValuesForModule(newModuleId);
const oldModule = ModuleRepository.getModule(moduleId);
const newModule = ModuleRepository.getModule(newModuleId);
if (oldModule && newModule) {
@ -349,9 +361,9 @@ export const ExploreView = React.memo(function ExploreView() {
<MenuItem
icon={IconNames.DUPLICATE}
text="Copy last query"
disabled={!QUERY_HISTORY.length}
disabled={!QUERY_LOG.length()}
onClick={() => {
copy(QUERY_HISTORY[0]?.sqlQuery, { format: 'text/plain' });
copy(QUERY_LOG.getLastQuery()!, { format: 'text/plain' });
AppToaster.show({
message: `Copied query to clipboard`,
intent: Intent.SUCCESS,
@ -360,9 +372,9 @@ export const ExploreView = React.memo(function ExploreView() {
/>
<MenuItem
icon={IconNames.HISTORY}
text="Show query history"
text="Show query log"
onClick={() => {
setShownText(getFormattedQueryHistory());
setShownText(QUERY_LOG.getFormatted());
}}
/>
<MenuItem

10
web-console/src/views/explore-view/models/measure.ts
View File

@ -90,7 +90,7 @@ export class Measure extends ExpressionMeta {
}
switch (column.nativeType) {
case 'BIGINT':
case 'LONG':
case 'FLOAT':
case 'DOUBLE':
return [
@ -103,16 +103,16 @@ export class Measure extends ExpressionMeta {
new Measure({
expression: F.min(C(column.name)),
}),
new Measure({
expression: SqlFunction.countDistinct(C(column.name)),
}),
new Measure({
as: `P98 ${column.name}`,
expression: F('APPROX_QUANTILE_DS', C(column.name), 0.98),
}),
new Measure({
expression: SqlFunction.countDistinct(C(column.name)),
}),
];
case 'VARCHAR':
case 'STRING':
case 'COMPLEX':
case 'COMPLEX<hyperUnique>':
return [

1
web-console/src/views/explore-view/models/parameter.ts
View File

@ -85,6 +85,7 @@ export type TypedParameterDefinition<Type extends keyof ParameterTypes> = TypedE
| ParameterTypes[Type]
| ((querySource: QuerySource) => ParameterTypes[Type] | undefined);
sticky?: boolean;
required?: ModuleFunctor<boolean>;
description?: ModuleFunctor<string>;
placeholder?: string;

12
web-console/src/views/explore-view/models/query-source.ts
View File

@ -95,7 +95,9 @@ export class QuerySource {
let effectiveColumns = columns;
if (query.getSelectExpressionsArray().some(ex => ex instanceof SqlStar)) {
// The query has a star so carefully pick the columns that make sense
effectiveColumns = columns.filter(c => c.sqlType !== 'OTHER');
effectiveColumns = columns.filter(
c => c.sqlType !== 'OTHER' || c.nativeType === 'COMPLEX<json>',
);
}
let measures = Measure.extractQueryMeasures(query);
@ -179,6 +181,10 @@ export class QuerySource {
return this.measures.some(m => m.name === name);
}
public hasBaseTimeColumn(): boolean {
return this.baseColumns.some(column => column.isTimeColumn());
}
public getSourceExpressionForColumn(outputName: string): SqlExpression {
const selectExpressionsArray = this.query.getSelectExpressionsArray();
@ -224,12 +230,12 @@ export class QuerySource {
return noStarQuery.addSelect(newExpression);
}
public addColumnAfter(neighborName: string, newExpression: SqlExpression): SqlQuery {
public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
return noStarQuery.changeSelectExpressions(
noStarQuery
.getSelectExpressionsArray()
.flatMap(ex => (ex.getOutputName() === neighborName ? [ex, newExpression] : ex)),
.flatMap(ex => (ex.getOutputName() === neighborName ? [ex, ...newExpressions] : ex)),
);
}

1
web-console/src/views/explore-view/modules/grouping-table-module.tsx
View File

@ -105,6 +105,7 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
count: `Show '<count> values'`,
},
defaultValue: 'null',
sticky: true,
visible: ({ parameterValues }) => Boolean((parameterValues.showColumns || []).length),
},
pivotColumn: {

29
web-console/src/views/explore-view/modules/record-table-module.tsx
View File

@ -21,10 +21,9 @@ import React, { useMemo } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import {
calculateInitPageSize,
GenericOutputTable,
} from '../components/generic-output-table/generic-output-table';
import type { ColumnHint } from '../../../utils';
import { filterMap } from '../../../utils';
import { calculateInitPageSize, GenericOutputTable } from '../components';
import { ModuleRepository } from '../module-repository/module-repository';
import './record-table-module.scss';
@ -33,6 +32,7 @@ interface RecordTableParameterValues {
maxRows: number;
ascending: boolean;
showTypeIcons: boolean;
hideNullColumns: boolean;
}
ModuleRepository.registerModule<RecordTableParameterValues>({
@ -50,10 +50,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
ascending: {
type: 'boolean',
defaultValue: false,
sticky: true,
},
showTypeIcons: {
type: 'boolean',
defaultValue: true,
sticky: true,
},
hideNullColumns: {
type: 'boolean',
label: 'Hide all null columns',
defaultValue: false,
sticky: true,
},
},
component: function RecordTableModule(props) {
@ -77,6 +85,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
});
const resultData = resultState.getSomeData();
let columnHints: Map<string, ColumnHint> | undefined;
if (parameterValues.hideNullColumns && resultData) {
columnHints = new Map<string, ColumnHint>(
filterMap(resultData.header, (column, i) =>
resultData.getColumnByIndex(i)?.every(v => v == null)
? [column.name, { hidden: true }]
: undefined,
),
);
}
return (
<div className="record-table-module module">
{resultState.error ? (
@ -84,6 +104,7 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
) : resultData ? (
<GenericOutputTable
queryResult={resultData}
columnHints={columnHints}
showTypeIcons={parameterValues.showTypeIcons}
onWhereChange={setWhere}
initPageSize={calculateInitPageSize(stage.height)}

3
web-console/src/views/explore-view/modules/time-chart-module.tsx
View File

@ -114,8 +114,9 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
},
snappyHighlight: {
type: 'boolean',
label: 'Snap highlight to nearest dates',
label: 'Snap highlight to granularity',
defaultValue: true,
sticky: true,
},
},
component: function TimeChartModule(props) {

10
web-console/src/views/explore-view/query-macros/aggregate.ts
View File

@ -20,6 +20,7 @@ import { C, SqlFunction, SqlQuery } from '@druid-toolkit/query';
import { filterMap, uniq } from '../../../utils';
import { Measure } from '../models';
import { KNOWN_AGGREGATIONS } from '../utils';
export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery {
const usedMeasures: Map<string, boolean> = new Map();
@ -35,7 +36,14 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery
if (!measure) throw new Error(`${Measure.AGGREGATE} of unknown measure '${measureName}'`);
usedMeasures.set(measureName, true);
return measure.expression;
let measureExpression = measure.expression;
const filter = ex.getWhereExpression();
if (filter) {
measureExpression = measureExpression.addFilterToAggregations(filter, KNOWN_AGGREGATIONS);
}
return measureExpression;
}
// If we encounter a (the) query with the measure definitions, and we have used those measures then expand out all the columns within them

2
web-console/src/views/explore-view/utils/index.ts
View File

@ -21,8 +21,10 @@ export * from './duration';
export * from './filter-pattern-helpers';
export * from './general';
export * from './get-auto-granularity';
export * from './known-aggregations';
export * from './max-time-for-table';
export * from './misc';
export * from './query-log';
export * from './snap-to-granularity';
export * from './table-query';
export * from './time-manipulation';

59
web-console/src/views/explore-view/utils/known-aggregations.ts Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const KNOWN_AGGREGATIONS = [
'COUNT',
'SUM',
'MIN',
'MAX',
'AVG',
'APPROX_COUNT_DISTINCT',
'APPROX_COUNT_DISTINCT_BUILTIN',
'APPROX_QUANTILE',
'APPROX_QUANTILE_FIXED_BUCKETS',
'BLOOM_FILTER',
'VAR_POP',
'VAR_SAMP',
'VARIANCE',
'STDDEV_POP',
'STDDEV_SAMP',
'STDDEV',
'EARLIEST',
'EARLIEST_BY',
'LATEST',
'LATEST_BY',
'ANY_VALUE',
'GROUPING',
'ARRAY_AGG',
'ARRAY_AGG',
'ARRAY_CONCAT_AGG',
'ARRAY_CONCAT_AGG',
'STRING_AGG',
'LISTAGG',
'BIT_AND',
'BIT_OR',
'BIT_XOR',
'APPROX_COUNT_DISTINCT_DS_THETA',
'DS_THETA',
'APPROX_QUANTILE_DS',
'DS_QUANTILES_SKETCH',
'DS_TUPLE_DOUBLES',
'DS_TUPLE_DOUBLES',
'TDIGEST_QUANTILE',
'TDIGEST_GENERATE_SKETCH',
];

48
web-console/src/views/explore-view/utils/query-log.ts Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
interface QueryLogEntry {
time: Date;
sqlQuery: string;
}
const MAX_QUERIES_TO_LOG = 10;
export class QueryLog {
private readonly queryLog: QueryLogEntry[] = [];
public length(): number {
return this.queryLog.length;
}
public addQuery(sqlQuery: string): void {
const { queryLog } = this;
queryLog.unshift({ time: new Date(), sqlQuery });
while (queryLog.length > MAX_QUERIES_TO_LOG) queryLog.pop();
}
public getLastQuery(): string | undefined {
return this.queryLog[0]?.sqlQuery;
}
public getFormatted(): string {
return this.queryLog
.map(({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`)
.join('\n\n-----------------------------------------------------\n\n');
}
}

30
web-console/src/views/explore-view/utils/table-query.ts
View File

@ -37,6 +37,7 @@ import { Measure } from '../models';
import { formatDuration } from './duration';
import { addTableScope } from './general';
import { KNOWN_AGGREGATIONS } from './known-aggregations';
import type { Compare } from './time-manipulation';
import { computeWhereForCompares } from './time-manipulation';
@ -48,35 +49,6 @@ export type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPerce
export type RestrictTop = 'always' | 'never';
const KNOWN_AGGREGATIONS = [
'COUNT',
'SUM',
'MIN',
'MAX',
'AVG',
'APPROX_COUNT_DISTINCT',
'APPROX_COUNT_DISTINCT_DS_HLL',
'APPROX_COUNT_DISTINCT_DS_THETA',
'DS_HLL',
'DS_THETA',
'APPROX_QUANTILE',
'APPROX_QUANTILE_DS',
'APPROX_QUANTILE_FIXED_BUCKETS',
'DS_QUANTILES_SKETCH',
'BLOOM_FILTER',
'TDIGEST_QUANTILE',
'TDIGEST_GENERATE_SKETCH',
'VAR_POP',
'VAR_SAMP',
'VARIANCE',
'STDDEV_POP',
'STDDEV_SAMP',
'STDDEV',
'EARLIEST',
'LATEST',
'ANY_VALUE',
];
const DRUID_DEFAULT_TOTAL_SUB_QUERY_LIMIT = 100000;
const COMMON_NAME = 'common';

15
web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
View File

@ -9,6 +9,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": false,
"queryType": "none",
@ -21,6 +22,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": false,
"queryType": "none",
@ -32,6 +34,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": false,
"queryType": "none",
@ -44,6 +47,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": false,
"queryType": "none",
@ -55,6 +59,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": false,
"queryType": "none",
@ -73,6 +78,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -85,6 +91,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -96,6 +103,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -109,6 +117,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -120,6 +129,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -132,6 +142,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -143,6 +154,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
@ -161,6 +173,7 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
Capabilities {
"coordinator": false,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": true,
"queryType": "none",
@ -173,6 +186,7 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
Capabilities {
"coordinator": false,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": true,
"queryType": "none",
@ -184,6 +198,7 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
Capabilities {
"coordinator": false,
"maxTaskSlots": undefined,
"multiStageQueryDart": false,
"multiStageQueryTask": false,
"overlord": true,
"queryType": "none",

4
web-console/src/views/workbench-view/column-tree/column-tree.tsx
View File

@ -688,10 +688,10 @@ export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeS
};
render() {
const { columnMetadataLoading } = this.props;
const { columnMetadata, columnMetadataLoading } = this.props;
const { currentSchemaSubtree, searchString } = this.state;
if (columnMetadataLoading) {
if (columnMetadataLoading && !columnMetadata) {
return (
<div className="column-tree">
<Loader />

121
web-console/src/views/workbench-view/current-dart-panel/current-dart-panel.scss Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../variables';
.current-dart-panel {
position: relative;
@include card-like;
overflow: auto;
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
.title {
position: relative;
border-bottom: 1px solid rgba(255, 255, 255, 0.3);
padding: 8px 10px;
user-select: none;
.close-button {
position: absolute;
top: 2px;
right: 2px;
}
}
.work-entries {
position: absolute;
top: 30px;
left: 0;
right: 0;
bottom: 0;
padding: 10px;
&:empty:after {
content: 'No current queries';
position: absolute;
top: 45%;
left: 50%;
transform: translate(-50%, -50%);
}
.work-entry {
display: block;
border-bottom: 1px solid rgba(255, 255, 255, 0.3);
padding-top: 8px;
padding-bottom: 8px;
cursor: pointer;
&:hover {
background-color: rgba(255, 255, 255, 0.1);
}
.line1 {
margin-bottom: 4px;
.status-icon {
display: inline-block;
margin-right: 5px;
&.running {
svg {
animation-name: spin;
animation-duration: 10s;
animation-iteration-count: infinite;
animation-timing-function: linear;
}
}
}
.timing {
display: inline-block;
}
}
.line2 {
white-space: nowrap;
overflow: hidden;
}
.identity-icon {
opacity: 0.6;
}
.identity-identity {
margin-left: 5px;
display: inline-block;
&.anonymous {
font-style: italic;
}
}
.query-indicator {
display: inline-block;
margin-left: 10px;
}
}
}
}

194
web-console/src/views/workbench-view/current-dart-panel/current-dart-panel.tsx Normal file
View File

@ -0,0 +1,194 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button, Icon, Intent, Menu, MenuDivider, MenuItem, Popover } from '@blueprintjs/core';
import { type IconName, IconNames } from '@blueprintjs/icons';
import classNames from 'classnames';
import copy from 'copy-to-clipboard';
import React, { useCallback, useState } from 'react';
import { useStore } from 'zustand';
import { Loader } from '../../../components';
import type { DartQueryEntry } from '../../../druid-models';
import { useClock, useInterval, useQueryManager } from '../../../hooks';
import { Api, AppToaster } from '../../../singletons';
import { formatDuration, prettyFormatIsoDate } from '../../../utils';
import { CancelQueryDialog } from '../cancel-query-dialog/cancel-query-dialog';
import { DartDetailsDialog } from '../dart-details-dialog/dart-details-dialog';
import { workStateStore } from '../work-state-store';
import './current-dart-panel.scss';
function stateToIconAndColor(status: DartQueryEntry['state']): [IconName, string] {
switch (status) {
case 'RUNNING':
return [IconNames.REFRESH, '#2167d5'];
case 'ACCEPTED':
return [IconNames.CIRCLE, '#8d8d8d'];
case 'CANCELED':
return [IconNames.DISABLE, '#8d8d8d'];
default:
return [IconNames.CIRCLE, '#8d8d8d'];
}
}
export interface CurrentViberPanelProps {
onClose(): void;
}
export const CurrentDartPanel = React.memo(function CurrentViberPanel(
props: CurrentViberPanelProps,
) {
const { onClose } = props;
const [showSql, setShowSql] = useState<string | undefined>();
const [confirmCancelId, setConfirmCancelId] = useState<string | undefined>();
const workStateVersion = useStore(
workStateStore,
useCallback(state => state.version, []),
);
const [dartQueryEntriesState, queryManager] = useQueryManager<number, DartQueryEntry[]>({
query: workStateVersion,
processQuery: async _ => {
return (await Api.instance.get('/druid/v2/sql/dart')).data.queries;
},
});
useInterval(() => {
queryManager.rerunLastQuery(true);
}, 3000);
const now = useClock();
const dartQueryEntries = dartQueryEntriesState.getSomeData();
return (
<div className="current-dart-panel">
<div className="title">
Current Dart queries
<Button className="close-button" icon={IconNames.CROSS} minimal onClick={onClose} />
</div>
{dartQueryEntries ? (
<div className="work-entries">
{dartQueryEntries.map(w => {
const menu = (
<Menu>
<MenuItem
icon={IconNames.EYE_OPEN}
text="Show SQL"
onClick={() => {
setShowSql(w.sql);
}}
/>
<MenuItem
icon={IconNames.DUPLICATE}
text="Copy SQL ID"
onClick={() => {
copy(w.sqlQueryId, { format: 'text/plain' });
AppToaster.show({
message: `${w.sqlQueryId} copied to clipboard`,
intent: Intent.SUCCESS,
});
}}
/>
<MenuItem
icon={IconNames.DUPLICATE}
text="Copy Dart ID"
onClick={() => {
copy(w.dartQueryId, { format: 'text/plain' });
AppToaster.show({
message: `${w.dartQueryId} copied to clipboard`,
intent: Intent.SUCCESS,
});
}}
/>
<MenuDivider />
<MenuItem
icon={IconNames.CROSS}
text="Cancel query"
intent={Intent.DANGER}
onClick={() => setConfirmCancelId(w.sqlQueryId)}
/>
</Menu>
);
const duration = now.valueOf() - new Date(w.startTime).valueOf();
const [icon, color] = stateToIconAndColor(w.state);
const anonymous = w.identity === 'allowAll' && w.authenticator === 'allowAll';
return (
<Popover className="work-entry" key={w.sqlQueryId} position="left" content={menu}>
<div>
<div className="line1">
<Icon
className={'status-icon ' + w.state.toLowerCase()}
icon={icon}
style={{ color }}
data-tooltip={`State: ${w.state}`}
/>
<div className="timing">
{prettyFormatIsoDate(w.startTime) +
((w.state === 'RUNNING' || w.state === 'ACCEPTED') && duration > 0
? ` (${formatDuration(duration)})`
: '')}
</div>
</div>
<div className="line2">
<Icon className="identity-icon" icon={IconNames.MUGSHOT} />
<div
className={classNames('identity-identity', { anonymous })}
data-tooltip={`Identity: ${w.identity}\nAuthenticator: ${w.authenticator}`}
>
{anonymous ? 'anonymous' : `${w.identity} (${w.authenticator})`}
</div>
</div>
</div>
</Popover>
);
})}
</div>
) : dartQueryEntriesState.isLoading() ? (
<Loader />
) : undefined}
{confirmCancelId && (
<CancelQueryDialog
// eslint-disable-next-line @typescript-eslint/no-misused-promises
onCancel={async () => {
if (!confirmCancelId) return;
try {
await Api.instance.delete(`/druid/v2/sql/dart/${Api.encodePath(confirmCancelId)}`);
AppToaster.show({
message: 'Query canceled',
intent: Intent.SUCCESS,
});
} catch {
AppToaster.show({
message: 'Could not cancel query',
intent: Intent.DANGER,
});
}
}}
onDismiss={() => setConfirmCancelId(undefined)}
/>
)}
{showSql && <DartDetailsDialog sql={showSql} onClose={() => setShowSql(undefined)} />}
</div>
);
});

35
web-console/src/views/workbench-view/dart-details-dialog/dart-details-dialog.scss Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../variables';
.dart-details-dialog {
&.#{$bp-ns}-dialog {
width: 95vw;
}
.#{$bp-ns}-dialog-body {
height: 70vh;
position: relative;
margin: 0;
.flexible-query-input {
height: 100%;
}
}
}

48
web-console/src/views/workbench-view/dart-details-dialog/dart-details-dialog.tsx Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button, Classes, Dialog } from '@blueprintjs/core';
import React from 'react';
import { FlexibleQueryInput } from '../flexible-query-input/flexible-query-input';
import './dart-details-dialog.scss';
export interface DartDetailsDialogProps {
sql: string;
onClose(): void;
}
export const DartDetailsDialog = React.memo(function DartDetailsDialog(
props: DartDetailsDialogProps,
) {
const { sql, onClose } = props;
return (
<Dialog className="dart-details-dialog" isOpen onClose={onClose} title="Dart SQL">
<div className={Classes.DIALOG_BODY}>
<FlexibleQueryInput queryString={sql} leaveBackground />
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Button text="Close" onClick={onClose} />
</div>
</div>
</Dialog>
);
});

6
web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap
View File

@ -134,12 +134,12 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
<span
className="cpu-label"
>
counter
Counter
</span>
<span
className="cpu-counter"
>
wall time
Wall time
</span>
</i>
</React.Fragment>,
@ -147,7 +147,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"className": "padded",
"id": "cpu",
"show": false,
"width": 220,
"width": 240,
},
{
"Header": <React.Fragment>

2
web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.scss
View File

@ -129,7 +129,7 @@
.cpu-label {
display: inline-block;
width: 120px;
width: 140px;
}
.cpu-counter {

10
web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
View File

@ -263,8 +263,8 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
Header: twoLines(
'CPU utilization',
<i>
<span className="cpu-label">counter</span>
<span className="cpu-counter">wall time</span>
<span className="cpu-label">Counter</span>
<span className="cpu-counter">Wall time</span>
</i>,
),
id: 'cpu',
@ -863,14 +863,14 @@ ${title} uncompressed size: ${formatBytesCompact(
Header: twoLines(
'CPU utilization',
<i>
<span className="cpu-label">counter</span>
<span className="cpu-counter">wall time</span>
<span className="cpu-label">Counter</span>
<span className="cpu-counter">Wall time</span>
</i>,
),
id: 'cpu',
accessor: () => null,
className: 'padded',
width: 220,
width: 240,
show: stages.hasCounter('cpu'),
Cell({ original }) {
const cpuTotals = stages.getCpuTotalsForStage(original);

2
web-console/src/views/workbench-view/execution-summary-panel/execution-summary-panel.tsx
View File

@ -96,7 +96,7 @@ export const ExecutionSummaryPanel = React.memo(function ExecutionSummaryPanel(
}
onClick={() => {
if (!execution) return;
if (oneOf(execution.engine, 'sql-msq-task')) {
if (oneOf(execution.engine, 'sql-msq-task', 'sql-msq-dart')) {
onExecutionDetail();
}
}}

5
web-console/src/views/workbench-view/explain-dialog/explain-dialog.tsx
View File

@ -45,6 +45,7 @@ import {
getDruidErrorMessage,
nonEmptyArray,
queryDruidSql,
queryDruidSqlDart,
} from '../../../utils';
import './explain-dialog.scss';
@ -108,6 +109,10 @@ export const ExplainDialog = React.memo(function ExplainDialog(props: ExplainDia
}
break;
case 'sql-msq-dart':
result = await queryDruidSqlDart(payload);
break;
default:
throw new Error(`Explain not supported for engine ${engine}`);
}

78
web-console/src/views/workbench-view/query-tab/query-tab.tsx
View File

@ -18,7 +18,7 @@
import { Code, Intent } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { QueryRunner, SqlQuery } from '@druid-toolkit/query';
import { QueryResult, QueryRunner, SqlQuery } from '@druid-toolkit/query';
import axios from 'axios';
import type { JSX } from 'react';
import React, { useCallback, useEffect, useRef, useState } from 'react';
@ -41,6 +41,7 @@ import type { WorkbenchRunningPromise } from '../../../singletons/workbench-runn
import { WorkbenchRunningPromises } from '../../../singletons/workbench-running-promises';
import type { ColumnMetadata, QueryAction, QuerySlice, RowColumn } from '../../../utils';
import {
deepGet,
DruidError,
findAllSqlQueriesInText,
localStorageGet,
@ -271,6 +272,67 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
return execution;
}
case 'sql-msq-dart': {
if (cancelQueryId) {
void cancelToken.promise
.then(cancel => {
if (cancel.message === QueryManager.TERMINATION_MESSAGE) return;
return Api.instance.delete(`/druid/v2/sql/dart/${Api.encodePath(cancelQueryId)}`);
})
.catch(() => {});
}
onQueryChange(props.query.changeLastExecution(undefined));
const executionPromise = Api.instance
.post(`/druid/v2/sql/dart`, query, {
cancelToken: new axios.CancelToken(cancelFn => {
nativeQueryCancelFnRef.current = cancelFn;
}),
})
.then(
({ data: dartResponse }) => {
if (deepGet(query, 'context.fullReport') && dartResponse[0][0] === 'fullReport') {
const dartReport = dartResponse[dartResponse.length - 1][0];
return Execution.fromTaskReport(dartReport)
.changeEngine('sql-msq-dart')
.changeSqlQuery(query.query, query.context);
} else {
return Execution.fromResult(
engine,
QueryResult.fromRawResult(
dartResponse,
false,
query.header,
query.typesHeader,
query.sqlTypesHeader,
),
).changeSqlQuery(query.query, query.context);
}
},
e => {
throw new DruidError(e, prefixLines);
},
);
WorkbenchRunningPromises.storePromise(id, {
executionPromise,
startTime,
});
let execution: Execution;
try {
execution = await executionPromise;
nativeQueryCancelFnRef.current = undefined;
} catch (e) {
nativeQueryCancelFnRef.current = undefined;
throw e;
}
return execution;
}
}
} else if (WorkbenchRunningPromises.isWorkbenchRunningPromise(q)) {
return await q.executionPromise;
@ -463,13 +525,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
</div>
)}
{execution &&
(execution.result ? (
<ResultTablePane
runeMode={execution.engine === 'native'}
queryResult={execution.result}
onQueryAction={handleQueryAction}
/>
) : execution.error ? (
(execution.error ? (
<div className="error-container">
<ExecutionErrorPane execution={execution} />
{execution.stages && (
@ -481,6 +537,12 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
/>
)}
</div>
) : execution.result ? (
<ResultTablePane
runeMode={execution.engine === 'native'}
queryResult={execution.result}
onQueryAction={handleQueryAction}
/>
) : execution.isSuccessfulIngest() ? (
<IngestSuccessPane
execution={execution}

1
web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx
View File

@ -225,6 +225,7 @@ LIMIT 100`,
className={'status-icon ' + w.taskStatus.toLowerCase()}
icon={icon}
style={{ color }}
data-tooltip={`Task status: ${w.taskStatus}`}
/>
<div className="timing">
{prettyFormatIsoDate(w.createdTime) +

4
web-console/src/views/workbench-view/run-panel/__snapshots__/run-panel.spec.tsx.snap
View File

@ -46,7 +46,7 @@ exports[`RunPanel matches snapshot on msq (auto) query 1`] = `
<span
class="bp5-button-text"
>
Engine: SQL MSQ-task
Engine: SQL (task)
</span>
<span
aria-hidden="true"
@ -150,7 +150,7 @@ exports[`RunPanel matches snapshot on native (auto) query 1`] = `
<span
class="bp5-button-text"
>
Engine: Auto (SQL native)
Engine: Auto [SQL (native)]
</span>
<span
aria-hidden="true"

374
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -103,13 +103,16 @@ const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
if (!engine) return { text: 'Auto' };
switch (engine) {
case 'native':
return { text: 'Native' };
return { text: 'JSON (native)' };
case 'sql-native':
return { text: 'SQL native' };
return { text: 'SQL (native)' };
case 'sql-msq-task':
return { text: 'SQL MSQ-task', label: 'multi-stage-query' };
return { text: 'SQL (task)', label: 'multi-stage-query' };
case 'sql-msq-dart':
return { text: 'SQL (Dart)', label: 'multi-stage-query' };
default:
return { text: engine };
@ -121,8 +124,6 @@ const SELECT_DESTINATION_LABEL: Record<SelectDestination, string> = {
durableStorage: 'Durable storage',
};
const EXPERIMENTAL_ICON = <Icon icon={IconNames.WARNING_SIGN} title="Experimental" />;
export type EnginesMenuOption =
| 'edit-query-context'
| 'define-parameters'
@ -135,7 +136,6 @@ export type EnginesMenuOption =
| 'finalize-aggregations'
| 'group-by-enable-multi-value-unnesting'
| 'durable-shuffle-storage'
| 'include-all-counters'
| 'use-cache'
| 'approximate-top-n'
| 'limit-inline-results';
@ -158,21 +158,24 @@ function optionVisible(
case 'finalize-aggregations':
case 'group-by-enable-multi-value-unnesting':
case 'durable-shuffle-storage':
case 'include-all-counters':
case 'join-algorithm':
return engine === 'sql-msq-task';
case 'join-algorithm':
return engine === 'sql-msq-task' || engine === 'sql-msq-dart';
case 'timezone':
case 'approximate-count-distinct':
return engine === 'sql-native' || engine === 'sql-msq-task';
return engine === 'sql-native' || engine === 'sql-msq-task' || engine === 'sql-msq-dart';
case 'use-cache':
return engine === 'native' || engine === 'sql-native';
case 'approximate-top-n':
case 'limit-inline-results':
return engine === 'sql-native';
case 'limit-inline-results':
return engine === 'sql-native' || engine === 'sql-msq-dart';
default:
console.warn(`Unknown option: ${option}`);
return false;
@ -251,16 +254,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
queryContext,
defaultQueryContext,
);
const useConcurrentLocks = getQueryContextKey(
'useConcurrentLocks',
queryContext,
defaultQueryContext,
);
const forceSegmentSortByTime = getQueryContextKey(
'forceSegmentSortByTime',
queryContext,
defaultQueryContext,
);
const finalizeAggregations = queryContext.finalizeAggregations;
const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
const groupByEnableMultiValueUnnesting = queryContext.groupByEnableMultiValueUnnesting;
@ -279,11 +272,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
queryContext,
defaultQueryContext,
);
const includeAllCounters = getQueryContextKey(
'includeAllCounters',
queryContext,
defaultQueryContext,
);
const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
@ -385,7 +373,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
<Menu>
{queryEngines.length > 1 && (
<>
<MenuDivider title="Select engine" />
<MenuDivider title="Select language and engine" />
<MenuItem
key="auto"
icon={tickIcon(queryEngine === undefined)}
@ -469,81 +457,33 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
/>
</MenuItem>
)}
{show('insert-replace-specific-context') && (
<MenuItem icon={IconNames.BRING_DATA} text="INSERT / REPLACE specific context">
<MenuBoolean
text="Force segment sort by time"
value={forceSegmentSortByTime}
onValueChange={forceSegmentSortByTime =>
changeQueryContext({
...queryContext,
forceSegmentSortByTime,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
/>
<MenuBoolean
text="Use concurrent locks"
value={useConcurrentLocks}
onValueChange={useConcurrentLocks =>
changeQueryContext({
...queryContext,
useConcurrentLocks,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
optionsLabelElement={{ true: EXPERIMENTAL_ICON }}
/>
<MenuBoolean
text="Fail on empty insert"
value={failOnEmptyInsert}
showUndefined
undefinedEffectiveValue={false}
onValueChange={failOnEmptyInsert =>
changeQueryContext({ ...queryContext, failOnEmptyInsert })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuBoolean
text="Wait until segments have loaded"
value={waitUntilSegmentsLoad}
showUndefined
undefinedEffectiveValue={ingestMode}
onValueChange={waitUntilSegmentsLoad =>
changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuItem
text="Edit index spec..."
label={summarizeIndexSpec(indexSpec)}
shouldDismissPopover={false}
onClick={() => {
setIndexSpecDialogSpec(indexSpec || {});
}}
/>
</MenuItem>
{show('approximate-count-distinct') && (
<MenuBoolean
icon={IconNames.ROCKET_SLANT}
text="Approximate COUNT(DISTINCT)"
value={useApproximateCountDistinct}
onValueChange={useApproximateCountDistinct =>
changeQueryContext({
...queryContext,
useApproximateCountDistinct,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('max-parse-exceptions') && (
<MenuItem
icon={IconNames.ERROR}
text="Max parse exceptions"
label={String(maxParseExceptions)}
>
{[0, 1, 5, 10, 1000, 10000, -1].map(v => (
<MenuItem
key={String(v)}
icon={tickIcon(v === maxParseExceptions)}
text={v === -1 ? '∞ (-1)' : String(v)}
onClick={() =>
changeQueryContext({ ...queryContext, maxParseExceptions: v })
}
shouldDismissPopover={false}
/>
))}
</MenuItem>
{show('approximate-top-n') && (
<MenuBoolean
icon={IconNames.HORIZONTAL_BAR_CHART_DESC}
text="Approximate TopN"
value={useApproximateTopN}
onValueChange={useApproximateTopN =>
changeQueryContext({
...queryContext,
useApproximateTopN,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('join-algorithm') && (
<MenuItem
@ -566,6 +506,125 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
</MenuItem>
)}
{show('insert-replace-specific-context') && (
<MenuItem
icon={IconNames.BRING_DATA}
text="INSERT / REPLACE / EXTERN specific context"
>
<MenuBoolean
text="Fail on empty insert"
value={failOnEmptyInsert}
showUndefined
undefinedEffectiveValue={false}
onValueChange={failOnEmptyInsert =>
changeQueryContext({ ...queryContext, failOnEmptyInsert })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuBoolean
text="Wait until segments have loaded"
value={waitUntilSegmentsLoad}
showUndefined
undefinedEffectiveValue={ingestMode}
onValueChange={waitUntilSegmentsLoad =>
changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuItem text="Max parse exceptions" label={String(maxParseExceptions)}>
{[0, 1, 5, 10, 1000, 10000, -1].map(v => (
<MenuItem
key={String(v)}
icon={tickIcon(v === maxParseExceptions)}
text={v === -1 ? '∞ (-1)' : String(v)}
onClick={() =>
changeQueryContext({ ...queryContext, maxParseExceptions: v })
}
shouldDismissPopover={false}
/>
))}
</MenuItem>
<MenuItem
text="Edit index spec..."
label={summarizeIndexSpec(indexSpec)}
shouldDismissPopover={false}
onClick={() => {
setIndexSpecDialogSpec(indexSpec || {});
}}
/>
</MenuItem>
)}
{show('finalize-aggregations') && (
<MenuBoolean
icon={IconNames.TRANSLATE}
text="Finalize aggregations"
value={finalizeAggregations}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={finalizeAggregations =>
changeQueryContext({ ...queryContext, finalizeAggregations })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('group-by-enable-multi-value-unnesting') && (
<MenuBoolean
icon={IconNames.FORK}
text="GROUP BY multi-value unnesting"
value={groupByEnableMultiValueUnnesting}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={groupByEnableMultiValueUnnesting =>
changeQueryContext({ ...queryContext, groupByEnableMultiValueUnnesting })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('use-cache') && (
<MenuBoolean
icon={IconNames.DATA_CONNECTION}
text="Use cache"
value={useCache}
onValueChange={useCache =>
changeQueryContext({
...queryContext,
useCache,
populateCache: useCache,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('limit-inline-results') && (
<MenuCheckbox
checked={!query.unlimited}
intent={query.unlimited ? Intent.WARNING : undefined}
text="Limit inline results"
labelElement={
query.unlimited ? <Icon icon={IconNames.WARNING_SIGN} /> : undefined
}
onChange={() => {
onQueryChange(query.toggleUnlimited());
}}
/>
)}
{show('durable-shuffle-storage') && (
<MenuBoolean
icon={IconNames.CLOUD_TICK}
text="Durable shuffle storage"
value={durableShuffleStorage}
onValueChange={durableShuffleStorage =>
changeQueryContext({
...queryContext,
durableShuffleStorage,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('select-destination') && (
<MenuItem
icon={IconNames.MANUALLY_ENTERED_DATA}
@ -602,119 +661,6 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
/>
</MenuItem>
)}
{show('finalize-aggregations') && (
<MenuBoolean
icon={IconNames.TRANSLATE}
text="Finalize aggregations"
value={finalizeAggregations}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={finalizeAggregations =>
changeQueryContext({ ...queryContext, finalizeAggregations })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('group-by-enable-multi-value-unnesting') && (
<MenuBoolean
icon={IconNames.FORK}
text="GROUP BY multi-value unnesting"
value={groupByEnableMultiValueUnnesting}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={groupByEnableMultiValueUnnesting =>
changeQueryContext({ ...queryContext, groupByEnableMultiValueUnnesting })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('durable-shuffle-storage') && (
<MenuBoolean
icon={IconNames.CLOUD_TICK}
text="Durable shuffle storage"
value={durableShuffleStorage}
onValueChange={durableShuffleStorage =>
changeQueryContext({
...queryContext,
durableShuffleStorage,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('use-cache') && (
<MenuBoolean
icon={IconNames.DATA_CONNECTION}
text="Use cache"
value={useCache}
onValueChange={useCache =>
changeQueryContext({
...queryContext,
useCache,
populateCache: useCache,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('approximate-top-n') && (
<MenuBoolean
icon={IconNames.HORIZONTAL_BAR_CHART_DESC}
text="Approximate TopN"
value={useApproximateTopN}
onValueChange={useApproximateTopN =>
changeQueryContext({
...queryContext,
useApproximateTopN,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('approximate-count-distinct') && (
<MenuBoolean
icon={IconNames.ROCKET_SLANT}
text="Approximate COUNT(DISTINCT)"
value={useApproximateCountDistinct}
onValueChange={useApproximateCountDistinct =>
changeQueryContext({
...queryContext,
useApproximateCountDistinct,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{show('limit-inline-results') && (
<MenuCheckbox
checked={!query.unlimited}
intent={query.unlimited ? Intent.WARNING : undefined}
text="Limit inline results"
labelElement={
query.unlimited ? <Icon icon={IconNames.WARNING_SIGN} /> : undefined
}
onChange={() => {
onQueryChange(query.toggleUnlimited());
}}
/>
)}
{show('include-all-counters') && (
<MenuBoolean
icon={IconNames.DIAGNOSIS}
text="Include all counters"
value={includeAllCounters}
onValueChange={includeAllCounters =>
changeQueryContext({
...queryContext,
includeAllCounters,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
</Menu>
}
>
@ -722,7 +668,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
text={`Engine: ${
queryEngine
? enginesLabelFn(queryEngine).text
: `${autoEngineLabel.text} (${enginesLabelFn(effectiveEngine).text})`
: `${autoEngineLabel.text} [${enginesLabelFn(effectiveEngine).text}]`
}`}
rightIcon={IconNames.CARET_DOWN}
intent={intent}

2
web-console/src/views/workbench-view/workbench-view.scss
View File

@ -45,7 +45,7 @@ $recent-query-task-panel-width: 250px;
gap: 2px;
.recent-query-task-panel,
.current-viper-panel {
.current-dart-panel {
flex: 1;
}
}

68
web-console/src/views/workbench-view/workbench-view.tsx
View File

@ -32,6 +32,7 @@ import classNames from 'classnames';
import copy from 'copy-to-clipboard';
import React from 'react';
import { MenuCheckbox } from '../../components';
import { SpecDialog, StringInputDialog } from '../../dialogs';
import type {
CapacityInfo,
@ -69,6 +70,7 @@ import {
import { ColumnTree } from './column-tree/column-tree';
import { ConnectExternalDataDialog } from './connect-external-data-dialog/connect-external-data-dialog';
import { CurrentDartPanel } from './current-dart-panel/current-dart-panel';
import { getDemoQueries } from './demo-queries';
import { ExecutionDetailsDialog } from './execution-details-dialog/execution-details-dialog';
import type { ExecutionDetailsTab } from './execution-details-pane/execution-details-pane';
@ -148,6 +150,7 @@ export interface WorkbenchViewState {
renamingTab?: TabEntry;
showRecentQueryTaskPanel: boolean;
showCurrentDartPanel: boolean;
}
export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, WorkbenchViewState> {
@ -166,6 +169,11 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
hasSqlTask && localStorageGetJson(LocalStorageKeys.WORKBENCH_TASK_PANEL),
);
const showCurrentDartPanel = Boolean(
queryEngines.includes('sql-msq-dart') &&
localStorageGetJson(LocalStorageKeys.WORKBENCH_DART_PANEL),
);
const tabEntries =
Array.isArray(possibleTabEntries) && possibleTabEntries.length
? possibleTabEntries.map(q => ({ ...q, query: new WorkbenchQuery(q.query) }))
@ -198,6 +206,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
taskIdSubmitDialogOpen: false,
showRecentQueryTaskPanel,
showCurrentDartPanel,
};
}
@ -264,6 +273,11 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
localStorageSetJson(LocalStorageKeys.WORKBENCH_TASK_PANEL, false);
};
private readonly handleCurrentDartPanelClose = () => {
this.setState({ showCurrentDartPanel: false });
localStorageSetJson(LocalStorageKeys.WORKBENCH_DART_PANEL, false);
};
private readonly handleDetailsWithId = (id: string, initTab?: ExecutionDetailsTab) => {
this.setState({
details: { id, initTab },
@ -656,7 +670,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
if (!queryEngines.includes('sql-msq-task')) return;
if (hideToolbar) return;
const { showRecentQueryTaskPanel } = this.state;
const { showRecentQueryTaskPanel, showCurrentDartPanel } = this.state;
return (
<ButtonGroup className="toolbar">
<Button
@ -669,16 +683,35 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
}}
minimal
/>
<Button
icon={IconNames.DRAWER_RIGHT}
minimal
data-tooltip="Open recent query task panel"
onClick={() => {
const n = !showRecentQueryTaskPanel;
this.setState({ showRecentQueryTaskPanel: n });
localStorageSetJson(LocalStorageKeys.WORKBENCH_TASK_PANEL, n);
}}
/>
<Popover
position="bottom-right"
content={
<Menu>
<MenuCheckbox
text="Recent query task panel"
checked={showRecentQueryTaskPanel}
shouldDismissPopover
onChange={() => {
const n = !showRecentQueryTaskPanel;
this.setState({ showRecentQueryTaskPanel: n });
localStorageSetJson(LocalStorageKeys.WORKBENCH_TASK_PANEL, n);
}}
/>
<MenuCheckbox
text="Current Dart query panel"
checked={showCurrentDartPanel}
shouldDismissPopover
onChange={() => {
const n = !showCurrentDartPanel;
this.setState({ showCurrentDartPanel: n });
localStorageSetJson(LocalStorageKeys.WORKBENCH_DART_PANEL, n);
}}
/>
</Menu>
}
>
<Button icon={IconNames.DRAWER_RIGHT} minimal data-tooltip="Open helper panels" />
</Popover>
</ButtonGroup>
);
}
@ -744,7 +777,9 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
runMoreMenu={
<Menu>
{!hiddenMoreMenuItems.includes('explain') &&
(effectiveEngine === 'sql-native' || effectiveEngine === 'sql-msq-task') && (
(effectiveEngine === 'sql-native' ||
effectiveEngine === 'sql-msq-task' ||
effectiveEngine === 'sql-msq-dart') && (
<MenuItem
icon={IconNames.CLEAN}
text="Explain SQL query"
@ -861,7 +896,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
};
render() {
const { columnMetadataState, showRecentQueryTaskPanel } = this.state;
const { columnMetadataState, showRecentQueryTaskPanel, showCurrentDartPanel } = this.state;
const query = this.getCurrentQuery();
let defaultSchema: string | undefined;
@ -872,7 +907,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
defaultTables = parsedQuery.getUsedTableNames();
}
const showRightPanel = showRecentQueryTaskPanel;
const showRightPanel = showRecentQueryTaskPanel || showCurrentDartPanel;
return (
<div
className={classNames('workbench-view app-view', {
@ -883,8 +918,8 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
{!columnMetadataState.isError() && (
<ColumnTree
getParsedQuery={this.getParsedQuery}
columnMetadata={columnMetadataState.getSomeData()}
columnMetadataLoading={columnMetadataState.loading}
columnMetadata={columnMetadataState.data}
onQueryChange={this.handleSqlQueryChange}
defaultSchema={defaultSchema ? defaultSchema : 'druid'}
defaultTables={defaultTables}
@ -903,6 +938,9 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
onNewTab={this.handleNewTab}
/>
)}
{showCurrentDartPanel && (
<CurrentDartPanel onClose={this.handleCurrentDartPanelClose} />
)}
</div>
)}
{this.renderExecutionDetailsDialog()}