Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

explore QA (#17225) (#17229)

This commit is contained in:
Vadim Ogievetsky 2024-10-03 01:00:46 -07:00 committed by GitHub
parent 13a52de07f
commit e4d1848aba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 154 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
web-console/src/views/explore-view/components/module-pane/module-pane.scss
View File

@ -35,9 +35,17 @@
& > .issue {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
}
.loader {
position: absolute;
top: 0;
left: 0;
}
}
.tile-content {

23
web-console/src/views/explore-view/explore-view.tsx
View File

@ -22,6 +22,7 @@ import { Button, Intent, Menu, MenuDivider, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { Column, QueryResult, SqlExpression } from '@druid-toolkit/query';
import { QueryRunner, SqlQuery } from '@druid-toolkit/query';
import type { CancelToken } from 'axios';
import classNames from 'classnames';
import copy from 'copy-to-clipboard';
import React, { useEffect, useMemo, useRef, useState } from 'react';
@ -79,27 +80,33 @@ const queryRunner = new QueryRunner({
},
});
async function runSqlQuery(query: string | SqlQuery): Promise<QueryResult> {
async function runSqlQuery(
query: string | SqlQuery,
cancelToken?: CancelToken,
): Promise<QueryResult> {
try {
return await queryRunner.runQuery({
query,
defaultQueryContext: {
sqlStringifyArrays: false,
},
cancelToken,
});
} catch (e) {
throw new DruidError(e);
}
}
async function introspectSource(source: string): Promise<QuerySource> {
async function introspectSource(source: string, cancelToken?: CancelToken): Promise<QuerySource> {
const query = SqlQuery.parse(source);
const introspectResult = await runSqlQuery(QuerySource.makeLimitZeroIntrospectionQuery(query));
cancelToken?.throwIfRequested();
const baseIntrospectResult = QuerySource.isSingleStarQuery(query)
? introspectResult
: await runSqlQuery(
QuerySource.makeLimitZeroIntrospectionQuery(QuerySource.stripToBaseSource(query)),
cancelToken,
);
return QuerySource.fromIntrospectResult(
@ -238,11 +245,15 @@ export const ExploreView = React.memo(function ExploreView() {
const querySource = querySourceState.getSomeData();
const runSqlPlusQuery = useMemo(() => {
return async (query: string | SqlQuery) => {
return async (query: string | SqlQuery, cancelToken?: CancelToken) => {
if (!querySource) throw new Error('no querySource');
return await runSqlQuery(
await rewriteMaxDataTime(rewriteAggregate(SqlQuery.parse(query), querySource.measures)),
);
const parsedQuery = SqlQuery.parse(query);
return (
await runSqlQuery(
await rewriteMaxDataTime(rewriteAggregate(parsedQuery, querySource.measures)),
cancelToken,
)
).attachQuery({ query: '' }, parsedQuery);
};
}, [querySource]);

72
web-console/src/views/explore-view/models/query-source.ts
View File

@ -44,31 +44,6 @@ export class QuerySource {
);
}
static materializeStarIfNeeded(query: SqlQuery, columns: readonly Column[]): SqlQuery {
let columnsToExpand = columns.map(c => c.name);
const selectExpressions = query.getSelectExpressionsArray();
let starCount = 0;
for (const selectExpression of selectExpressions) {
if (selectExpression instanceof SqlStar) {
starCount++;
continue;
}
const outputName = selectExpression.getOutputName();
if (!outputName) continue;
columnsToExpand = columnsToExpand.filter(c => c !== outputName);
}
if (starCount === 0) return query;
if (starCount > 1) throw new Error('can not handle multiple stars');
return query
.changeSelectExpressions(
selectExpressions.flatMap(selectExpression =>
selectExpression instanceof SqlStar ? columnsToExpand.map(c => C(c)) : selectExpression,
),
)
.prettify();
}
static isSingleStarQuery(query: SqlQuery): boolean {
const selectExpressions = query.getSelectExpressionsArray();
return selectExpressions.length === 1 && selectExpressions[0] instanceof SqlStar;
@ -151,6 +126,35 @@ export class QuerySource {
};
}
private materializeStarIfNeeded(): SqlQuery {
const { query, columns, measures } = this;
let columnsToExpand = columns.map(c => c.name);
const selectExpressions = query.getSelectExpressionsArray();
let starCount = 0;
for (const selectExpression of selectExpressions) {
if (selectExpression instanceof SqlStar) {
starCount++;
continue;
}
const outputName = selectExpression.getOutputName();
if (!outputName) continue;
columnsToExpand = columnsToExpand.filter(c => c !== outputName);
}
if (starCount === 0) return query;
if (starCount > 1) throw new Error('can not handle multiple stars');
return Measure.addMeasuresToQuery(
query
.changeSelectExpressions(
selectExpressions.flatMap(selectExpression =>
selectExpression instanceof SqlStar ? columnsToExpand.map(c => C(c)) : selectExpression,
),
)
.prettify(),
measures,
);
}
public getFirstAggregateMeasure(): Measure | undefined {
return this.measures[0]?.toAggregateBasedMeasure();
}
@ -226,12 +230,12 @@ export class QuerySource {
}
public addColumn(newExpression: SqlExpression): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return noStarQuery.addSelect(newExpression);
}
public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return noStarQuery.changeSelectExpressions(
noStarQuery
.getSelectExpressionsArray()
@ -240,7 +244,7 @@ export class QuerySource {
}
public changeColumn(oldName: string, newExpression: SqlExpression): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return noStarQuery.changeSelectExpressions(
noStarQuery
.getSelectExpressionsArray()
@ -249,7 +253,7 @@ export class QuerySource {
}
public deleteColumn(outputName: string): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return noStarQuery.changeSelectExpressions(
noStarQuery.getSelectExpressionsArray().filter(ex => ex.getOutputName() !== outputName),
);
@ -260,7 +264,7 @@ export class QuerySource {
}
public applyColumnNameMap(columnNameMap: Map<string, string>): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return noStarQuery.changeSelectExpressions(
noStarQuery.getSelectExpressionsArray().map(ex => {
const outputName = ex.getOutputName();
@ -275,12 +279,12 @@ export class QuerySource {
// ------------------------------------
public addMeasure(measure: Measure): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return Measure.addMeasuresToQuery(noStarQuery, this.measures.concat(measure));
}
public addMeasureAfter(neighborName: string, newMeasure: Measure): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return Measure.addMeasuresToQuery(
noStarQuery,
this.measures.flatMap(m => (m.name === neighborName ? [m, newMeasure] : m)),
@ -288,7 +292,7 @@ export class QuerySource {
}
public changeMeasure(oldName: string, newMeasure: Measure): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return Measure.addMeasuresToQuery(
noStarQuery,
this.measures.map(m => (m.name === oldName ? newMeasure : m)),
@ -296,7 +300,7 @@ export class QuerySource {
}
public deleteMeasure(measureName: string): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
const noStarQuery = this.materializeStarIfNeeded();
return Measure.addMeasuresToQuery(
noStarQuery,
this.measures.filter(m => m.name !== measureName),

3
web-console/src/views/explore-view/module-repository/module-repository.ts
View File

@ -17,6 +17,7 @@
*/
import type { QueryResult, SqlExpression, SqlQuery } from '@druid-toolkit/query';
import type { CancelToken } from 'axios';
import type { ParameterDefinition, QuerySource, Stage } from '../models';
@ -34,7 +35,7 @@ interface ModuleComponentProps<P> {
setWhere(where: SqlExpression): void;
parameterValues: P;
setParameterValues: (parameters: Partial<P>) => void;
runSqlQuery(query: string | SqlQuery): Promise<QueryResult>;
runSqlQuery(query: string | SqlQuery, cancelToken?: CancelToken): Promise<QueryResult>;
}
export class ModuleRepository {

10
web-console/src/views/explore-view/modules/bar-chart-module.tsx
View File

@ -21,6 +21,7 @@ import type { ECharts } from 'echarts';
import * as echarts from 'echarts';
import React, { useEffect, useMemo, useRef } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import { formatEmpty } from '../../../utils';
import { Issue } from '../components';
@ -90,10 +91,10 @@ ModuleRepository.registerModule<BarChartParameterValues>({
.changeLimitValue(limit);
}, [querySource, where, splitColumn, measure, measureToSort, limit]);
const [sourceDataState] = useQueryManager({
const [sourceDataState, queryManager] = useQueryManager({
query: dataQuery,
processQuery: async (query: SqlQuery) => {
return (await runSqlQuery(query)).toObjectArray();
processQuery: async (query, cancelToken) => {
return (await runSqlQuery(query, cancelToken)).toObjectArray();
},
});
@ -203,6 +204,9 @@ ModuleRepository.registerModule<BarChartParameterValues>({
}}
/>
{errorMessage && <Issue issue={errorMessage} />}
{sourceDataState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

68
web-console/src/views/explore-view/modules/grouping-table-module.tsx
View File

@ -17,7 +17,7 @@
*/
import { Button } from '@blueprintjs/core';
import type { SqlOrderByDirection } from '@druid-toolkit/query';
import type { SqlExpression, SqlOrderByDirection } from '@druid-toolkit/query';
import { C, F, SqlQuery } from '@druid-toolkit/query';
import React, { useMemo } from 'react';
@ -43,6 +43,11 @@ import './grouping-table-module.scss';
// when ordering on non __time is more robust
const NEEDS_GROUPING_TO_ORDER = true;
interface QueryAndMore {
originalWhere: SqlExpression;
queryAndHints: QueryAndHints;
}
interface GroupingTableParameterValues {
splitColumns: ExpressionMeta[];
timeBucket: string;
@ -216,14 +221,14 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
.changeLimitValue(maxPivotValues);
}, [querySource.query, parameterValues]);
const [pivotValueState] = useQueryManager({
const [pivotValueState, queryManager] = useQueryManager({
query: pivotValueQuery,
processQuery: async (pivotValueQuery: SqlQuery) => {
return (await runSqlQuery(pivotValueQuery)).getColumnByName('v') as string[];
},
});
const queryAndHints = useMemo((): QueryAndHints | undefined => {
const queryAndMore = useMemo((): QueryAndMore | undefined => {
const pivotValues = pivotValueState.data;
if (parameterValues.pivotColumn && !pivotValues) return;
const { orderByColumn, orderByDirection } = parameterValues;
@ -231,32 +236,43 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
? C(orderByColumn).toOrderByExpression(orderByDirection)
: undefined;
return makeTableQueryAndHints({
source: querySource.query,
where,
splitColumns: parameterValues.splitColumns,
timeBucket: parameterValues.timeBucket,
showColumns: parameterValues.showColumns,
multipleValueMode: parameterValues.multipleValueMode,
pivotColumn: parameterValues.pivotColumn,
pivotValues,
measures: parameterValues.measures,
compares: parameterValues.compares || [],
compareStrategy: parameterValues.compareStrategy,
compareTypes: parameterValues.compareTypes,
restrictTop: parameterValues.restrictTop,
maxRows: parameterValues.maxRows,
orderBy,
useGroupingToOrderSubQueries: NEEDS_GROUPING_TO_ORDER,
});
return {
originalWhere: where,
queryAndHints: makeTableQueryAndHints({
source: querySource.query,
where,
splitColumns: parameterValues.splitColumns,
timeBucket: parameterValues.timeBucket,
showColumns: parameterValues.showColumns,
multipleValueMode: parameterValues.multipleValueMode,
pivotColumn: parameterValues.pivotColumn,
pivotValues,
measures: parameterValues.measures,
compares: parameterValues.compares || [],
compareStrategy: parameterValues.compareStrategy,
compareTypes: parameterValues.compareTypes,
restrictTop: parameterValues.restrictTop,
maxRows: parameterValues.maxRows,
orderBy,
useGroupingToOrderSubQueries: NEEDS_GROUPING_TO_ORDER,
}),
};
}, [querySource.query, where, parameterValues, pivotValueState.data]);
const [resultState] = useQueryManager({
query: queryAndHints,
processQuery: async (queryAndHints: QueryAndHints) => {
query: queryAndMore,
processQuery: async (queryAndMore, cancelToken) => {
const { originalWhere, queryAndHints } = queryAndMore;
const { query, columnHints } = queryAndHints;
let result = await runSqlQuery(query, cancelToken);
if (result.sqlQuery) {
result = result.attachQuery(
{ query: '' },
result.sqlQuery.changeWhereExpression(originalWhere),
);
}
return {
result: await runSqlQuery(query),
result,
columnHints,
};
},
@ -297,7 +313,9 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
initPageSize={calculateInitPageSize(stage.height)}
/>
) : undefined}
{resultState.loading && <Loader />}
{resultState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

12
web-console/src/views/explore-view/modules/multi-axis-chart-module.tsx
View File

@ -21,6 +21,7 @@ import type { ECharts } from 'echarts';
import * as echarts from 'echarts';
import React, { useEffect, useMemo, useRef } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import {
formatInteger,
@ -91,16 +92,16 @@ ModuleRepository.registerModule<MultiAxisChartParameterValues>({
direction: 'ASC',
})
.applyForEach(measures, (q, measure) => q.addSelect(measure.expression.as(measure.name)));
}, [querySource, where, timeGranularity, measures]);
}, [querySource, where, timeColumnName, timeGranularity, measures]);
const [sourceDataState] = useQueryManager({
const [sourceDataState, queryManager] = useQueryManager({
query: dataQuery,
processQuery: async (query: SqlQuery) => {
processQuery: async (query: SqlQuery, cancelToken) => {
if (!timeColumnName) {
throw new Error(`Must have a column of type TIMESTAMP for the multi-axis chart to work`);
}
return (await runSqlQuery(query)).toObjectArray();
return (await runSqlQuery(query, cancelToken)).toObjectArray();
},
});
@ -327,6 +328,9 @@ ModuleRepository.registerModule<MultiAxisChartParameterValues>({
}}
/>
{errorMessage && <Issue issue={errorMessage} />}
{sourceDataState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

10
web-console/src/views/explore-view/modules/pie-chart-module.tsx
View File

@ -21,6 +21,7 @@ import type { ECharts } from 'echarts';
import * as echarts from 'echarts';
import React, { useEffect, useMemo, useRef } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import { formatEmpty, formatNumber } from '../../../utils';
import { Issue } from '../components';
@ -113,10 +114,10 @@ ModuleRepository.registerModule<PieChartParameterValues>({
};
}, [querySource, where, splitColumn, measure, limit, showOthers]);
const [sourceDataState] = useQueryManager({
const [sourceDataState, queryManager] = useQueryManager({
query: dataQueries,
processQuery: async ({ mainQuery, splitExpression, othersPartialQuery }) => {
const result = await runSqlQuery(mainQuery);
processQuery: async ({ mainQuery, splitExpression, othersPartialQuery }, cancelToken) => {
const result = await runSqlQuery(mainQuery, cancelToken);
const data = result.toObjectArray();
if (splitExpression && othersPartialQuery) {
@ -251,6 +252,9 @@ ModuleRepository.registerModule<PieChartParameterValues>({
}}
/>
{errorMessage && <Issue issue={errorMessage} />}
{sourceDataState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

6
web-console/src/views/explore-view/modules/record-table-module.tsx
View File

@ -79,7 +79,7 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
.toString();
}, [querySource, where, parameterValues]);
const [resultState] = useQueryManager({
const [resultState, queryManager] = useQueryManager({
query: query,
processQuery: runSqlQuery,
});
@ -110,7 +110,9 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
initPageSize={calculateInitPageSize(stage.height)}
/>
) : undefined}
{resultState.loading && <Loader />}
{resultState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

15
web-console/src/views/explore-view/modules/time-chart-module.tsx
View File

@ -21,6 +21,7 @@ import type { ECharts } from 'echarts';
import * as echarts from 'echarts';
import React, { useEffect, useMemo, useRef } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import {
formatInteger,
@ -141,9 +142,12 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
};
}, [querySource, where, measure, splitColumn, numberToStack, showOthers]);
const [sourceDataState] = useQueryManager({
const [sourceDataState, queryManager] = useQueryManager({
query: dataQuery,
processQuery: async ({ baseQuery, measure, splitExpression, numberToStack, showOthers }) => {
processQuery: async (
{ baseQuery, measure, splitExpression, numberToStack, showOthers },
cancelToken,
) => {
if (!timeColumnName) {
throw new Error(`Must have a column of type TIMESTAMP for the time chart to work`);
}
@ -155,10 +159,13 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
.addSelect(splitExpression.as('v'), { addToGroupBy: 'end' })
.changeOrderByExpression(measure.expression.toOrderByExpression('DESC'))
.changeLimitValue(numberToStack),
cancelToken,
)
).getColumnByIndex(0)!
: undefined;
cancelToken.throwIfRequested();
const dataset = (
await runSqlQuery(
baseQuery
@ -181,6 +188,7 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
);
})
.addSelect(measure.expression.as(METRIC_NAME)),
cancelToken,
)
).toObjectArray();
@ -430,6 +438,9 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
}}
/>
{errorMessage && <Issue issue={errorMessage} />}
{sourceDataState.loading && (
<Loader cancelText="Cancel query" onCancel={() => queryManager.cancelCurrent()} />
)}
</div>
);
},

2
web-console/src/views/explore-view/query-macros/aggregate.ts
View File

@ -55,7 +55,7 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery
filterMap(queryMeasures, queryMeasure =>
usedMeasures.get(queryMeasure.name) ? queryMeasure.expression : undefined,
).flatMap(ex => ex.getUsedColumnNames()),
).filter(columnName => !ex.getSelectIndexForOutputColumn(columnName)),
).filter(columnName => ex.getSelectIndexForOutputColumn(columnName) === -1),
(q, columnName) => q.addSelect(C(columnName)),
);
}

8
web-console/webpack.config.js
View File

@ -32,8 +32,12 @@ function friendlyErrorFormatter(e) {
module.exports = env => {
let druidUrl = (env || {}).druid_host || process.env.druid_host || 'localhost';
if (!druidUrl.startsWith('http')) druidUrl = 'http://' + druidUrl;
if (!/:\d+$/.test(druidUrl)) druidUrl += ':8888';
if (!druidUrl.startsWith('http')) {
druidUrl = (druidUrl.endsWith(':9088') ? 'https://' : 'http://') + druidUrl;
}
if (!/:\d+$/.test(druidUrl)) {
druidUrl += druidUrl.startsWith('https://') ? ':9088' : ':8888';
}
const proxyTarget = {
target: druidUrl,