Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: misc fixes to the Explore view (#17213) 

* make record table able to hide column

* stickyness

* refactor query log

* fix measure drag

* start nested column dialog

* nested expand

* fix filtering on Measures

* use output name

* fix scrolling

* select all / none

* use ARRAY_CONCAT_AGG

* no need to limit if aggregating

* remove magic number

* better search

* update arg list

* add, don't replace
This commit is contained in:
Vadim Ogievetsky 2024-10-02 08:52:08 -07:00 committed by GitHub
parent e5d027ee1c
commit 715ae5ece0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 529 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web-console/src/utils/local-storage-keys.tsx
View File

@ -58,6 +58,7 @@ export const LocalStorageKeys = {
SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const,
EXPLORE_STATE: 'explore-state' as const,
EXPLORE_STICKY: 'explore-sticky' as const,
};
export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys];

1
web-console/src/utils/table-helpers.ts
View File

@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination {
export interface ColumnHint {
displayName?: string;
group?: string;
hidden?: boolean;
expressionForWhere?: SqlExpression;
formatter?: (x: any) => string;
}

22
web-console/src/views/explore-view/components/control-pane/control-pane.tsx
View File

@ -194,7 +194,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
};
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<ExpressionMeta>
allowReordering
values={effectiveValue ? [effectiveValue] : []}
onValuesChange={vs => onValueChange(vs[0])}
@ -223,7 +223,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
);
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<ExpressionMeta>
allowReordering
values={effectiveValue as ExpressionMeta[]}
onValuesChange={onValueChange}
@ -266,7 +266,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
case 'measure': {
return {
element: (
<NamedExpressionsInput
<NamedExpressionsInput<Measure>
values={effectiveValue ? [effectiveValue] : []}
onValuesChange={vs => onValueChange(vs[0])}
singleton
@ -284,9 +284,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
/>
),
onDropColumn: column => {
const measures = Measure.getPossibleMeasuresForColumn(column);
if (!measures.length) return;
onValueChange(measures[0]);
const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue || effectiveValue.name !== p.name,
);
if (!candidateMeasures.length) return;
onValueChange(candidateMeasures[0]);
},
onDropMeasure: onValueChange,
};
@ -313,11 +315,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
/>
),
onDropColumn: column => {
const measures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue.some((v: ExpressionMeta) => v.name === p.name),
const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
p => !effectiveValue.some((v: Measure) => v.name === p.name),
);
if (!measures.length) return;
onValueChange(effectiveValue.concat(measures[0]));
if (!candidateMeasures.length) return;
onValueChange(effectiveValue.concat(candidateMeasures[0]));
},
onDropMeasure: measure => {
onValueChange(effectiveValue.concat(measure));

1
web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx
View File

@ -61,6 +61,7 @@ export const NamedExpressionsInput = function NamedExpressionsInput<
const onDragOver = useCallback(
(e: React.DragEvent, i: number) => {
if (dragIndex === -1) return;
const targetRect = e.currentTarget.getBoundingClientRect();
const before = e.clientX - targetRect.left <= targetRect.width / 2;
setDropBefore(before);

1
web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx
View File

@ -52,6 +52,7 @@ export const ContainsFilterControl = React.memo(function ContainsFilterControl(
),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
// eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
[querySource.query, filter, column, contains, negated],

1
web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx
View File

@ -58,6 +58,7 @@ export const RegexpFilterControl = React.memo(function RegexpFilterControl(
SqlExpression.and(filter, regexp ? filterPatternToExpression(filterPattern) : undefined),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
// eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
[querySource.query, filter, column, regexp, negated],

52
web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx
View File

@ -16,14 +16,15 @@
* limitations under the License.
*/
import { FormGroup, InputGroup, Menu, MenuItem } from '@blueprintjs/core';
import { FormGroup, Menu, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { QueryResult, SqlQuery, ValuesFilterPattern } from '@druid-toolkit/query';
import { C, F, L, SqlExpression, SqlLiteral } from '@druid-toolkit/query';
import type { QueryResult, ValuesFilterPattern } from '@druid-toolkit/query';
import { C, F, SqlExpression, SqlQuery } from '@druid-toolkit/query';
import React, { useMemo, useState } from 'react';
import { ClearableInput } from '../../../../../../components';
import { useQueryManager } from '../../../../../../hooks';
import { caseInsensitiveContains } from '../../../../../../utils';
import { caseInsensitiveContains, filterMap } from '../../../../../../utils';
import type { QuerySource } from '../../../../models';
import { toggle } from '../../../../utils';
import { ColumnValue } from '../../column-value/column-value';
@ -46,21 +47,21 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
const [initValues] = useState(selectedValues);
const [searchString, setSearchString] = useState('');
const valuesQuery = useMemo(() => {
const columnRef = C(column);
const queryParts: string[] = [`SELECT ${columnRef.as('c')}`, `FROM (${querySource.query})`];
const filterEx = SqlExpression.and(
const valuesQuery = useMemo(
() =>
SqlQuery.from(querySource.query)
.addSelect(C(column).as('c'), { addToGroupBy: 'end' })
.changeWhereExpression(
SqlExpression.and(
filter,
searchString ? F('ICONTAINS_STRING', columnRef, L(searchString)) : undefined,
searchString ? F('ICONTAINS_STRING', C(column), searchString) : undefined,
),
)
.changeOrderByExpression(F.count().toOrderByExpression('DESC'))
.changeLimitValue(101)
.toString(),
[querySource.query, filter, column, searchString],
);
if (!(filterEx instanceof SqlLiteral)) {
queryParts.push(`WHERE ${filterEx}`);
}
queryParts.push(`GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 101`);
return queryParts.join('\n');
}, [querySource.query, filter, column, searchString]);
const [valuesState] = useQueryManager<string, any[]>({
query: valuesQuery,
@ -77,23 +78,17 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
if (values) {
valuesToShow = valuesToShow.concat(values.filter(v => !initValues.includes(v)));
}
if (searchString) {
valuesToShow = valuesToShow.filter(v => caseInsensitiveContains(v, searchString));
}
const showSearch = querySource.columns.find(c => c.name === column)?.sqlType !== 'BOOLEAN';
return (
<FormGroup className="values-filter-control">
{showSearch && (
<InputGroup
value={searchString}
onChange={e => setSearchString(e.target.value)}
placeholder="Search"
/>
<ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
)}
<Menu className="value-list">
{valuesToShow.map((v, i) => (
{filterMap(valuesToShow, (v, i) => {
if (!caseInsensitiveContains(v, searchString)) return;
return (
<MenuItem
key={i}
icon={
@ -112,7 +107,8 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
});
}}
/>
))}
);
})}
{valuesState.loading && <MenuItem icon={IconNames.BLANK} text="Loading..." disabled />}
</Menu>
</FormGroup>

4
web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx
View File

@ -428,6 +428,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
columns={columnNester(
queryResult.header.map((column, i) => {
const h = column.name;
const hint = columnHints?.get(h);
const icon = showTypeIcons ? columnToIcon(column) : undefined;
return {
@ -446,9 +447,10 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
},
headerClassName: getHeaderClassName(h),
accessor: String(i),
show: !hint?.hidden,
Cell(row) {
const value = row.value;
const formatter = columnHints?.get(h)?.formatter || formatNumber;
const formatter = hint?.formatter || formatNumber;
return (
<div>
<Popover content={<Deferred content={() => getCellMenu(column, i, value)} />}>

7
web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx
View File

@ -48,9 +48,8 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
if (!expression) return;
return SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
.addSelect(F.cast(expression, 'VARCHAR').as('v'), { addToGroupBy: 'end' })
.applyIf(
querySource.baseColumns.find(column => column.isTimeColumn()),
q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
)
.changeLimitValue(100)
.toString();
@ -151,7 +150,7 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
} else {
onApply(
querySource.changeColumn(initExpressionName, newExpression),
new Map([[initExpression.getOutputName()!, newExpression.getOutputName()!]]),
new Map([[initExpressionName, newExpression.getOutputName()!]]),
);
}
} else {

5
web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx
View File

@ -57,9 +57,8 @@ export const MeasureDialog = React.memo(function MeasureDialog(props: MeasureDia
.changeWithParts([SqlWithPart.simple('t', QuerySource.stripToBaseSource(querySource.query))])
.addSelect(L('Overall').as('label'))
.addSelect(expression.as('value'))
.applyIf(
querySource.baseColumns.find(column => column.isTimeColumn()),
q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
)
.toString();
}, [querySource.query, formula]);

44
web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../../../variables';
.nested-column-dialog {
&.#{$bp-ns}-dialog {
width: 50vw;
min-height: 540px;
}
.#{$bp-ns}-dialog-body {
display: flex;
flex-direction: column;
.path-selector {
flex: 1;
padding: 5px 0;
height: 400px;
overflow: auto;
border-left: 1px solid rgba(15, 19, 32, 0.4);
border-right: 1px solid rgba(15, 19, 32, 0.4);
}
}
.#{$bp-ns}-dialog-footer {
margin-top: 0;
}
}

176
web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Button,
ButtonGroup,
Classes,
Dialog,
FormGroup,
InputGroup,
Intent,
Menu,
Tag,
} from '@blueprintjs/core';
import type { SqlExpression } from '@druid-toolkit/query';
import { type QueryResult, F, sql, SqlFunction, SqlQuery } from '@druid-toolkit/query';
import React, { useState } from 'react';
import { ClearableInput, Loader, MenuCheckbox } from '../../../../../components';
import { useQueryManager } from '../../../../../hooks';
import { caseInsensitiveContains, filterMap, pluralIfNeeded } from '../../../../../utils';
import { ExpressionMeta, QuerySource } from '../../../models';
import { toggle } from '../../../utils';
import './nested-column-dialog.scss';
const ARRAY_CONCAT_AGG_SIZE = 10000;
export interface NestedColumnDialogProps {
nestedColumn: SqlExpression;
onApply(newQuery: SqlQuery): void;
querySource: QuerySource;
runSqlQuery(query: string | SqlQuery): Promise<QueryResult>;
onClose(): void;
}
export const NestedColumnDialog = React.memo(function NestedColumnDialog(
props: NestedColumnDialogProps,
) {
const { nestedColumn, onApply, querySource, runSqlQuery, onClose } = props;
const [searchString, setSearchString] = useState('');
const [selectedPaths, setSelectedPaths] = useState<string[]>([]);
const [namingScheme, setNamingScheme] = useState(`${nestedColumn.getFirstColumnName()}[%]`);
const [pathsState] = useQueryManager({
query: nestedColumn,
processQuery: async nestedColumn => {
const query = SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
.addSelect(
SqlFunction.decorated('ARRAY_CONCAT_AGG', 'DISTINCT', [
F('JSON_PATHS', nestedColumn),
ARRAY_CONCAT_AGG_SIZE,
]),
)
.applyIf(querySource.hasBaseTimeColumn(), q =>
q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
);
const pathResult = await runSqlQuery(query);
const paths = pathResult.rows[0]?.[0];
if (!Array.isArray(paths)) throw new Error('Could not get paths');
return paths;
},
});
const paths = pathsState.data;
return (
<Dialog className="nested-column-dialog" isOpen onClose={onClose} title="Expand nested column">
<div className={Classes.DIALOG_BODY}>
<p>
Replace <Tag minimal>{String(nestedColumn.getOutputName())}</Tag> with path expansions for
the selected paths.
</p>
{pathsState.isLoading() && <Loader />}
{pathsState.getErrorMessage()}
{paths && (
<FormGroup>
<ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
<Menu className="path-selector">
{filterMap(paths, (path, i) => {
if (!caseInsensitiveContains(path, searchString)) return;
return (
<MenuCheckbox
key={i}
checked={selectedPaths.includes(path)}
onChange={() => setSelectedPaths(toggle(selectedPaths, path))}
text={path}
/>
);
})}
</Menu>
<ButtonGroup fill>
<Button
text="Select all"
onClick={() =>
// Select all paths matching the shown search string
setSelectedPaths(
paths.filter(path => caseInsensitiveContains(path, searchString)),
)
}
/>
<Button
text="Select none"
onClick={() =>
// Remove from selection all the paths matching the search string
setSelectedPaths(
selectedPaths.filter(path => !caseInsensitiveContains(path, searchString)),
)
}
/>
</ButtonGroup>
</FormGroup>
)}
<FormGroup label="Nameing scheme">
<InputGroup
value={namingScheme}
onChange={e => {
setNamingScheme(e.target.value.slice(0, ExpressionMeta.MAX_NAME_LENGTH));
}}
/>
</FormGroup>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<div className="edit-column-dialog-buttons">
<Button text="Cancel" onClick={onClose} />
<Button
text={
selectedPaths.length
? `Add ${pluralIfNeeded(selectedPaths.length, 'column')}`
: 'Select path'
}
disabled={!selectedPaths.length}
intent={Intent.PRIMARY}
onClick={() => {
const effectiveNamingScheme = namingScheme.includes('%')
? namingScheme
: namingScheme + '[%]';
onApply(
querySource.addColumnAfter(
nestedColumn.getOutputName()!,
...selectedPaths.map(path =>
F('JSON_VALUE', nestedColumn, path).as(
querySource.getAvailableName(
effectiveNamingScheme.replaceAll('%', path.replace(/^\$\./, '')),
),
),
),
),
);
onClose();
}}
/>
</div>
</div>
</div>
</Dialog>
);
});

30
web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx
View File

@ -40,6 +40,7 @@ import type { Rename } from '../../utils';
import { ColumnDialog } from './column-dialog/column-dialog';
import { MeasureDialog } from './measure-dialog/measure-dialog';
import { NestedColumnDialog } from './nested-column-dialog/nested-column-dialog';
import './resource-pane.scss';
@ -67,6 +68,9 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
const [columnSearch, setColumnSearch] = useState('');
const [columnEditorOpenOn, setColumnEditorOpenOn] = useState<ColumnEditorOpenOn | undefined>();
const [nestedColumnEditorOpenOn, setNestedColumnEditorOpenOn] = useState<
SqlExpression | undefined
>();
const [measureEditorOpenOn, setMeasureEditorOpenOn] = useState<MeasureEditorOpenOn | undefined>();
function applyUtil(nameTransform: (columnName: string) => string) {
@ -112,6 +116,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
<div className="column-resource-list">
{filterMap(querySource.columns, (column, i) => {
const columnName = column.name;
const isNestedColumn = column.nativeType === 'COMPLEX<json>';
if (!caseInsensitiveContains(columnName, columnSearch)) return;
return (
<Popover
@ -120,6 +125,18 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
position="right"
content={
<Menu>
{isNestedColumn ? (
<MenuItem
icon={IconNames.EXPAND_ALL}
text="Expand nested column"
onClick={() =>
setNestedColumnEditorOpenOn(
querySource.getSourceExpressionForColumn(columnName),
)
}
/>
) : (
<>
{onFilter && (
<MenuItem
icon={IconNames.FILTER}
@ -133,6 +150,8 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
onClick={() => onShowColumn(column)}
/>
<MenuDivider />
</>
)}
<MenuItem
icon={IconNames.EDIT}
text="Edit"
@ -165,7 +184,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
>
<div
className={Classes.MENU_ITEM}
draggable
draggable={!isNestedColumn}
onDragStart={e => {
e.dataTransfer.effectAllowed = 'all';
DragHelper.dragColumn = column;
@ -268,6 +287,15 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
onClose={() => setColumnEditorOpenOn(undefined)}
/>
)}
{nestedColumnEditorOpenOn && (
<NestedColumnDialog
nestedColumn={nestedColumnEditorOpenOn}
onApply={newQuery => onQueryChange(newQuery, undefined)}
querySource={querySource}
runSqlQuery={runSqlQuery}
onClose={() => setNestedColumnEditorOpenOn(undefined)}
/>
)}
{measureEditorOpenOn && (
<MeasureDialog
initMeasure={measureEditorOpenOn.measure}

66
web-console/src/views/explore-view/explore-view.tsx
View File

@ -30,7 +30,15 @@ import { useStore } from 'zustand';
import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
import { useHashAndLocalStorageHybridState, useQueryManager } from '../../hooks';
import { Api, AppToaster } from '../../singletons';
import { DruidError, LocalStorageKeys, queryDruidSql } from '../../utils';
import {
DruidError,
isEmpty,
localStorageGetJson,
LocalStorageKeys,
localStorageSetJson,
mapRecord,
queryDruidSql,
} from '../../utils';
import {
ControlPane,
@ -50,29 +58,14 @@ import { QuerySource } from './models';
import { ModuleRepository } from './module-repository/module-repository';
import { rewriteAggregate, rewriteMaxDataTime } from './query-macros';
import type { Rename } from './utils';
import { adjustTransferValue, normalizeType } from './utils';
import { adjustTransferValue, normalizeType, QueryLog } from './utils';
import './explore-view.scss';
// ---------------------------------------
const QUERY_LOG = new QueryLog();
interface QueryHistoryEntry {
time: Date;
sqlQuery: string;
}
const MAX_PAST_QUERIES = 10;
const QUERY_HISTORY: QueryHistoryEntry[] = [];
function addQueryToHistory(sqlQuery: string): void {
QUERY_HISTORY.unshift({ time: new Date(), sqlQuery });
while (QUERY_HISTORY.length > MAX_PAST_QUERIES) QUERY_HISTORY.pop();
}
function getFormattedQueryHistory(): string {
return QUERY_HISTORY.map(
({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`,
).join('\n\n-----------------------------------------------------\n\n');
function getStickyParameterValuesForModule(moduleId: string): ParameterValues {
return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {};
}
// ---------------------------------------
@ -81,7 +74,7 @@ const queryRunner = new QueryRunner({
inflateDateStrategy: 'fromSqlTypes',
executor: async (sqlQueryPayload, isSql, cancelToken) => {
if (!isSql) throw new Error('should never get here');
addQueryToHistory(sqlQueryPayload.query);
QUERY_LOG.addQuery(sqlQueryPayload.query);
return Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken });
},
});
@ -90,6 +83,9 @@ async function runSqlQuery(query: string | SqlQuery): Promise<QueryResult> {
try {
return await queryRunner.runQuery({
query,
defaultQueryContext: {
sqlStringifyArrays: false,
},
});
} catch (e) {
throw new DruidError(e);
@ -193,10 +189,25 @@ export const ExploreView = React.memo(function ExploreView() {
}
function resetParameterValues() {
setParameterValues({});
setParameterValues(getStickyParameterValuesForModule(moduleId));
}
function updateParameterValues(newParameterValues: ParameterValues) {
// Evaluate sticky-ness
if (module) {
const currentExploreSticky = localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY) || {};
const currentModuleSticky = currentExploreSticky[moduleId] || {};
const newModuleSticky = {
...currentModuleSticky,
...mapRecord(newParameterValues, (v, k) => (module.parameters[k]?.sticky ? v : undefined)),
};
localStorageSetJson(LocalStorageKeys.EXPLORE_STICKY, {
...currentExploreSticky,
[moduleId]: isEmpty(newModuleSticky) ? undefined : newModuleSticky,
});
}
setParameterValues({ ...parameterValues, ...newParameterValues });
}
@ -311,7 +322,8 @@ export const ExploreView = React.memo(function ExploreView() {
]}
selectedModuleId={moduleId}
onSelectedModuleIdChange={newModuleId => {
const newParameterValues: ParameterValues = {};
const newParameterValues = getStickyParameterValuesForModule(newModuleId);
const oldModule = ModuleRepository.getModule(moduleId);
const newModule = ModuleRepository.getModule(newModuleId);
if (oldModule && newModule) {
@ -349,9 +361,9 @@ export const ExploreView = React.memo(function ExploreView() {
<MenuItem
icon={IconNames.DUPLICATE}
text="Copy last query"
disabled={!QUERY_HISTORY.length}
disabled={!QUERY_LOG.length()}
onClick={() => {
copy(QUERY_HISTORY[0]?.sqlQuery, { format: 'text/plain' });
copy(QUERY_LOG.getLastQuery()!, { format: 'text/plain' });
AppToaster.show({
message: `Copied query to clipboard`,
intent: Intent.SUCCESS,
@ -360,9 +372,9 @@ export const ExploreView = React.memo(function ExploreView() {
/>
<MenuItem
icon={IconNames.HISTORY}
text="Show query history"
text="Show query log"
onClick={() => {
setShownText(getFormattedQueryHistory());
setShownText(QUERY_LOG.getFormatted());
}}
/>
<MenuItem

10
web-console/src/views/explore-view/models/measure.ts
View File

@ -90,7 +90,7 @@ export class Measure extends ExpressionMeta {
}
switch (column.nativeType) {
case 'BIGINT':
case 'LONG':
case 'FLOAT':
case 'DOUBLE':
return [
@ -103,16 +103,16 @@ export class Measure extends ExpressionMeta {
new Measure({
expression: F.min(C(column.name)),
}),
new Measure({
expression: SqlFunction.countDistinct(C(column.name)),
}),
new Measure({
as: `P98 ${column.name}`,
expression: F('APPROX_QUANTILE_DS', C(column.name), 0.98),
}),
new Measure({
expression: SqlFunction.countDistinct(C(column.name)),
}),
];
case 'VARCHAR':
case 'STRING':
case 'COMPLEX':
case 'COMPLEX<hyperUnique>':
return [

1
web-console/src/views/explore-view/models/parameter.ts
View File

@ -85,6 +85,7 @@ export type TypedParameterDefinition<Type extends keyof ParameterTypes> = TypedE
| ParameterTypes[Type]
| ((querySource: QuerySource) => ParameterTypes[Type] | undefined);
sticky?: boolean;
required?: ModuleFunctor<boolean>;
description?: ModuleFunctor<string>;
placeholder?: string;

12
web-console/src/views/explore-view/models/query-source.ts
View File

@ -95,7 +95,9 @@ export class QuerySource {
let effectiveColumns = columns;
if (query.getSelectExpressionsArray().some(ex => ex instanceof SqlStar)) {
// The query has a star so carefully pick the columns that make sense
effectiveColumns = columns.filter(c => c.sqlType !== 'OTHER');
effectiveColumns = columns.filter(
c => c.sqlType !== 'OTHER' || c.nativeType === 'COMPLEX<json>',
);
}
let measures = Measure.extractQueryMeasures(query);
@ -179,6 +181,10 @@ export class QuerySource {
return this.measures.some(m => m.name === name);
}
public hasBaseTimeColumn(): boolean {
return this.baseColumns.some(column => column.isTimeColumn());
}
public getSourceExpressionForColumn(outputName: string): SqlExpression {
const selectExpressionsArray = this.query.getSelectExpressionsArray();
@ -224,12 +230,12 @@ export class QuerySource {
return noStarQuery.addSelect(newExpression);
}
public addColumnAfter(neighborName: string, newExpression: SqlExpression): SqlQuery {
public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery {
const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
return noStarQuery.changeSelectExpressions(
noStarQuery
.getSelectExpressionsArray()
.flatMap(ex => (ex.getOutputName() === neighborName ? [ex, newExpression] : ex)),
.flatMap(ex => (ex.getOutputName() === neighborName ? [ex, ...newExpressions] : ex)),
);
}

1
web-console/src/views/explore-view/modules/grouping-table-module.tsx
View File

@ -105,6 +105,7 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
count: `Show '<count> values'`,
},
defaultValue: 'null',
sticky: true,
visible: ({ parameterValues }) => Boolean((parameterValues.showColumns || []).length),
},
pivotColumn: {

29
web-console/src/views/explore-view/modules/record-table-module.tsx
View File

@ -21,10 +21,9 @@ import React, { useMemo } from 'react';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import {
calculateInitPageSize,
GenericOutputTable,
} from '../components/generic-output-table/generic-output-table';
import type { ColumnHint } from '../../../utils';
import { filterMap } from '../../../utils';
import { calculateInitPageSize, GenericOutputTable } from '../components';
import { ModuleRepository } from '../module-repository/module-repository';
import './record-table-module.scss';
@ -33,6 +32,7 @@ interface RecordTableParameterValues {
maxRows: number;
ascending: boolean;
showTypeIcons: boolean;
hideNullColumns: boolean;
}
ModuleRepository.registerModule<RecordTableParameterValues>({
@ -50,10 +50,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
ascending: {
type: 'boolean',
defaultValue: false,
sticky: true,
},
showTypeIcons: {
type: 'boolean',
defaultValue: true,
sticky: true,
},
hideNullColumns: {
type: 'boolean',
label: 'Hide all null columns',
defaultValue: false,
sticky: true,
},
},
component: function RecordTableModule(props) {
@ -77,6 +85,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
});
const resultData = resultState.getSomeData();
let columnHints: Map<string, ColumnHint> | undefined;
if (parameterValues.hideNullColumns && resultData) {
columnHints = new Map<string, ColumnHint>(
filterMap(resultData.header, (column, i) =>
resultData.getColumnByIndex(i)?.every(v => v == null)
? [column.name, { hidden: true }]
: undefined,
),
);
}
return (
<div className="record-table-module module">
{resultState.error ? (
@ -84,6 +104,7 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
) : resultData ? (
<GenericOutputTable
queryResult={resultData}
columnHints={columnHints}
showTypeIcons={parameterValues.showTypeIcons}
onWhereChange={setWhere}
initPageSize={calculateInitPageSize(stage.height)}

3
web-console/src/views/explore-view/modules/time-chart-module.tsx
View File

@ -114,8 +114,9 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
},
snappyHighlight: {
type: 'boolean',
label: 'Snap highlight to nearest dates',
label: 'Snap highlight to granularity',
defaultValue: true,
sticky: true,
},
},
component: function TimeChartModule(props) {

10
web-console/src/views/explore-view/query-macros/aggregate.ts
View File

@ -20,6 +20,7 @@ import { C, SqlFunction, SqlQuery } from '@druid-toolkit/query';
import { filterMap, uniq } from '../../../utils';
import { Measure } from '../models';
import { KNOWN_AGGREGATIONS } from '../utils';
export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery {
const usedMeasures: Map<string, boolean> = new Map();
@ -35,7 +36,14 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery
if (!measure) throw new Error(`${Measure.AGGREGATE} of unknown measure '${measureName}'`);
usedMeasures.set(measureName, true);
return measure.expression;
let measureExpression = measure.expression;
const filter = ex.getWhereExpression();
if (filter) {
measureExpression = measureExpression.addFilterToAggregations(filter, KNOWN_AGGREGATIONS);
}
return measureExpression;
}
// If we encounter a (the) query with the measure definitions, and we have used those measures then expand out all the columns within them

2
web-console/src/views/explore-view/utils/index.ts
View File

@ -21,8 +21,10 @@ export * from './duration';
export * from './filter-pattern-helpers';
export * from './general';
export * from './get-auto-granularity';
export * from './known-aggregations';
export * from './max-time-for-table';
export * from './misc';
export * from './query-log';
export * from './snap-to-granularity';
export * from './table-query';
export * from './time-manipulation';

59
web-console/src/views/explore-view/utils/known-aggregations.ts Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const KNOWN_AGGREGATIONS = [
'COUNT',
'SUM',
'MIN',
'MAX',
'AVG',
'APPROX_COUNT_DISTINCT',
'APPROX_COUNT_DISTINCT_BUILTIN',
'APPROX_QUANTILE',
'APPROX_QUANTILE_FIXED_BUCKETS',
'BLOOM_FILTER',
'VAR_POP',
'VAR_SAMP',
'VARIANCE',
'STDDEV_POP',
'STDDEV_SAMP',
'STDDEV',
'EARLIEST',
'EARLIEST_BY',
'LATEST',
'LATEST_BY',
'ANY_VALUE',
'GROUPING',
'ARRAY_AGG',
'ARRAY_AGG',
'ARRAY_CONCAT_AGG',
'ARRAY_CONCAT_AGG',
'STRING_AGG',
'LISTAGG',
'BIT_AND',
'BIT_OR',
'BIT_XOR',
'APPROX_COUNT_DISTINCT_DS_THETA',
'DS_THETA',
'APPROX_QUANTILE_DS',
'DS_QUANTILES_SKETCH',
'DS_TUPLE_DOUBLES',
'DS_TUPLE_DOUBLES',
'TDIGEST_QUANTILE',
'TDIGEST_GENERATE_SKETCH',
];

48
web-console/src/views/explore-view/utils/query-log.ts Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
interface QueryLogEntry {
time: Date;
sqlQuery: string;
}
const MAX_QUERIES_TO_LOG = 10;
export class QueryLog {
private readonly queryLog: QueryLogEntry[] = [];
public length(): number {
return this.queryLog.length;
}
public addQuery(sqlQuery: string): void {
const { queryLog } = this;
queryLog.unshift({ time: new Date(), sqlQuery });
while (queryLog.length > MAX_QUERIES_TO_LOG) queryLog.pop();
}
public getLastQuery(): string | undefined {
return this.queryLog[0]?.sqlQuery;
}
public getFormatted(): string {
return this.queryLog
.map(({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`)
.join('\n\n-----------------------------------------------------\n\n');
}
}

30
web-console/src/views/explore-view/utils/table-query.ts
View File

@ -37,6 +37,7 @@ import { Measure } from '../models';
import { formatDuration } from './duration';
import { addTableScope } from './general';
import { KNOWN_AGGREGATIONS } from './known-aggregations';
import type { Compare } from './time-manipulation';
import { computeWhereForCompares } from './time-manipulation';
@ -48,35 +49,6 @@ export type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPerce
export type RestrictTop = 'always' | 'never';
const KNOWN_AGGREGATIONS = [
'COUNT',
'SUM',
'MIN',
'MAX',
'AVG',
'APPROX_COUNT_DISTINCT',
'APPROX_COUNT_DISTINCT_DS_HLL',
'APPROX_COUNT_DISTINCT_DS_THETA',
'DS_HLL',
'DS_THETA',
'APPROX_QUANTILE',
'APPROX_QUANTILE_DS',
'APPROX_QUANTILE_FIXED_BUCKETS',
'DS_QUANTILES_SKETCH',
'BLOOM_FILTER',
'TDIGEST_QUANTILE',
'TDIGEST_GENERATE_SKETCH',
'VAR_POP',
'VAR_SAMP',
'VARIANCE',
'STDDEV_POP',
'STDDEV_SAMP',
'STDDEV',
'EARLIEST',
'LATEST',
'ANY_VALUE',
];
const DRUID_DEFAULT_TOTAL_SUB_QUERY_LIMIT = 100000;
const COMMON_NAME = 'common';