From 715ae5ece0d6d6d410daf7c32720017db0a88031 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Oct 2024 08:52:08 -0700 Subject: [PATCH] Web console: misc fixes to the Explore view (#17213) * make record table able to hide column * stickyness * refactor query log * fix measure drag * start nested column dialog * nested expand * fix filtering on Measures * use output name * fix scrolling * select all / none * use ARRAY_CONCAT_AGG * no need to limit if aggregating * remove magic number * better search * update arg list * add, don't replace --- web-console/src/utils/local-storage-keys.tsx | 1 + web-console/src/utils/table-helpers.ts | 1 + .../components/control-pane/control-pane.tsx | 22 ++- .../control-pane/named-expressions-input.tsx | 1 + .../contains-filter-control.tsx | 1 + .../regexp-filter-control.tsx | 1 + .../values-filter-control.tsx | 92 +++++---- .../generic-output-table.tsx | 4 +- .../column-dialog/column-dialog.tsx | 7 +- .../measure-dialog/measure-dialog.tsx | 5 +- .../nested-column-dialog.scss | 44 +++++ .../nested-column-dialog.tsx | 176 ++++++++++++++++++ .../resource-pane/resource-pane.tsx | 50 +++-- .../src/views/explore-view/explore-view.tsx | 66 ++++--- .../src/views/explore-view/models/measure.ts | 10 +- .../views/explore-view/models/parameter.ts | 1 + .../views/explore-view/models/query-source.ts | 12 +- .../modules/grouping-table-module.tsx | 1 + .../modules/record-table-module.tsx | 29 ++- .../modules/time-chart-module.tsx | 3 +- .../explore-view/query-macros/aggregate.ts | 10 +- .../src/views/explore-view/utils/index.ts | 2 + .../explore-view/utils/known-aggregations.ts | 59 ++++++ .../src/views/explore-view/utils/query-log.ts | 48 +++++ .../views/explore-view/utils/table-query.ts | 30 +-- 25 files changed, 529 insertions(+), 147 deletions(-) create mode 100644 web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss create mode 100644 web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx create mode 100644 web-console/src/views/explore-view/utils/known-aggregations.ts create mode 100644 web-console/src/views/explore-view/utils/query-log.ts diff --git a/web-console/src/utils/local-storage-keys.tsx b/web-console/src/utils/local-storage-keys.tsx index 8a8fee96b20..a178fc2c009 100644 --- a/web-console/src/utils/local-storage-keys.tsx +++ b/web-console/src/utils/local-storage-keys.tsx @@ -58,6 +58,7 @@ export const LocalStorageKeys = { SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const, EXPLORE_STATE: 'explore-state' as const, + EXPLORE_STICKY: 'explore-sticky' as const, }; export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys]; diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts index 90df7fa1064..45e8758bf6f 100644 --- a/web-console/src/utils/table-helpers.ts +++ b/web-console/src/utils/table-helpers.ts @@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination { export interface ColumnHint { displayName?: string; group?: string; + hidden?: boolean; expressionForWhere?: SqlExpression; formatter?: (x: any) => string; } diff --git a/web-console/src/views/explore-view/components/control-pane/control-pane.tsx b/web-console/src/views/explore-view/components/control-pane/control-pane.tsx index 7fcc6352f29..faa7576bb67 100644 --- a/web-console/src/views/explore-view/components/control-pane/control-pane.tsx +++ b/web-console/src/views/explore-view/components/control-pane/control-pane.tsx @@ -194,7 +194,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { }; return { element: ( - allowReordering values={effectiveValue ? [effectiveValue] : []} onValuesChange={vs => onValueChange(vs[0])} @@ -223,7 +223,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { ); return { element: ( - allowReordering values={effectiveValue as ExpressionMeta[]} onValuesChange={onValueChange} @@ -266,7 +266,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { case 'measure': { return { element: ( - values={effectiveValue ? [effectiveValue] : []} onValuesChange={vs => onValueChange(vs[0])} singleton @@ -284,9 +284,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { /> ), onDropColumn: column => { - const measures = Measure.getPossibleMeasuresForColumn(column); - if (!measures.length) return; - onValueChange(measures[0]); + const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter( + p => !effectiveValue || effectiveValue.name !== p.name, + ); + if (!candidateMeasures.length) return; + onValueChange(candidateMeasures[0]); }, onDropMeasure: onValueChange, }; @@ -313,11 +315,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { /> ), onDropColumn: column => { - const measures = Measure.getPossibleMeasuresForColumn(column).filter( - p => !effectiveValue.some((v: ExpressionMeta) => v.name === p.name), + const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter( + p => !effectiveValue.some((v: Measure) => v.name === p.name), ); - if (!measures.length) return; - onValueChange(effectiveValue.concat(measures[0])); + if (!candidateMeasures.length) return; + onValueChange(effectiveValue.concat(candidateMeasures[0])); }, onDropMeasure: measure => { onValueChange(effectiveValue.concat(measure)); diff --git a/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx b/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx index ae498341ecd..d93a6c522ce 100644 --- a/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx +++ b/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx @@ -61,6 +61,7 @@ export const NamedExpressionsInput = function NamedExpressionsInput< const onDragOver = useCallback( (e: React.DragEvent, i: number) => { + if (dragIndex === -1) return; const targetRect = e.currentTarget.getBoundingClientRect(); const before = e.clientX - targetRect.left <= targetRect.width / 2; setDropBefore(before); diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx index 353179c905e..8751aad7e60 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx @@ -52,6 +52,7 @@ export const ContainsFilterControl = React.memo(function ContainsFilterControl( ), ) .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) .toString(), // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps [querySource.query, filter, column, contains, negated], diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx index ea6bdf74385..dd9a90d4bf6 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx @@ -58,6 +58,7 @@ export const RegexpFilterControl = React.memo(function RegexpFilterControl( SqlExpression.and(filter, regexp ? filterPatternToExpression(filterPattern) : undefined), ) .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) .toString(), // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps [querySource.query, filter, column, regexp, negated], diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx index a15d3daad76..d0d5f3f460a 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx @@ -16,14 +16,15 @@ * limitations under the License. */ -import { FormGroup, InputGroup, Menu, MenuItem } from '@blueprintjs/core'; +import { FormGroup, Menu, MenuItem } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; -import type { QueryResult, SqlQuery, ValuesFilterPattern } from '@druid-toolkit/query'; -import { C, F, L, SqlExpression, SqlLiteral } from '@druid-toolkit/query'; +import type { QueryResult, ValuesFilterPattern } from '@druid-toolkit/query'; +import { C, F, SqlExpression, SqlQuery } from '@druid-toolkit/query'; import React, { useMemo, useState } from 'react'; +import { ClearableInput } from '../../../../../../components'; import { useQueryManager } from '../../../../../../hooks'; -import { caseInsensitiveContains } from '../../../../../../utils'; +import { caseInsensitiveContains, filterMap } from '../../../../../../utils'; import type { QuerySource } from '../../../../models'; import { toggle } from '../../../../utils'; import { ColumnValue } from '../../column-value/column-value'; @@ -46,21 +47,21 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl( const [initValues] = useState(selectedValues); const [searchString, setSearchString] = useState(''); - const valuesQuery = useMemo(() => { - const columnRef = C(column); - const queryParts: string[] = [`SELECT ${columnRef.as('c')}`, `FROM (${querySource.query})`]; - - const filterEx = SqlExpression.and( - filter, - searchString ? F('ICONTAINS_STRING', columnRef, L(searchString)) : undefined, - ); - if (!(filterEx instanceof SqlLiteral)) { - queryParts.push(`WHERE ${filterEx}`); - } - - queryParts.push(`GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 101`); - return queryParts.join('\n'); - }, [querySource.query, filter, column, searchString]); + const valuesQuery = useMemo( + () => + SqlQuery.from(querySource.query) + .addSelect(C(column).as('c'), { addToGroupBy: 'end' }) + .changeWhereExpression( + SqlExpression.and( + filter, + searchString ? F('ICONTAINS_STRING', C(column), searchString) : undefined, + ), + ) + .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) + .toString(), + [querySource.query, filter, column, searchString], + ); const [valuesState] = useQueryManager({ query: valuesQuery, @@ -77,42 +78,37 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl( if (values) { valuesToShow = valuesToShow.concat(values.filter(v => !initValues.includes(v))); } - if (searchString) { - valuesToShow = valuesToShow.filter(v => caseInsensitiveContains(v, searchString)); - } const showSearch = querySource.columns.find(c => c.name === column)?.sqlType !== 'BOOLEAN'; - return ( {showSearch && ( - setSearchString(e.target.value)} - placeholder="Search" - /> + )} - {valuesToShow.map((v, i) => ( - } - shouldDismissPopover={false} - onClick={e => { - setFilterPattern({ - ...filterPattern, - values: e.altKey ? [v] : toggle(selectedValues, v), - }); - }} - /> - ))} + {filterMap(valuesToShow, (v, i) => { + if (!caseInsensitiveContains(v, searchString)) return; + return ( + } + shouldDismissPopover={false} + onClick={e => { + setFilterPattern({ + ...filterPattern, + values: e.altKey ? [v] : toggle(selectedValues, v), + }); + }} + /> + ); + })} {valuesState.loading && } diff --git a/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx b/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx index 0050a34122c..b557d99b4fd 100644 --- a/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx +++ b/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx @@ -428,6 +428,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( columns={columnNester( queryResult.header.map((column, i) => { const h = column.name; + const hint = columnHints?.get(h); const icon = showTypeIcons ? columnToIcon(column) : undefined; return { @@ -446,9 +447,10 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( }, headerClassName: getHeaderClassName(h), accessor: String(i), + show: !hint?.hidden, Cell(row) { const value = row.value; - const formatter = columnHints?.get(h)?.formatter || formatNumber; + const formatter = hint?.formatter || formatNumber; return (
getCellMenu(column, i, value)} />}> diff --git a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx index 4cc4fcd674c..0e4cc3cb77a 100644 --- a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx @@ -48,9 +48,8 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog if (!expression) return; return SqlQuery.from(QuerySource.stripToBaseSource(querySource.query)) .addSelect(F.cast(expression, 'VARCHAR').as('v'), { addToGroupBy: 'end' }) - .applyIf( - querySource.baseColumns.find(column => column.isTimeColumn()), - q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), ) .changeLimitValue(100) .toString(); @@ -151,7 +150,7 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog } else { onApply( querySource.changeColumn(initExpressionName, newExpression), - new Map([[initExpression.getOutputName()!, newExpression.getOutputName()!]]), + new Map([[initExpressionName, newExpression.getOutputName()!]]), ); } } else { diff --git a/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx index f02facf61f0..b48638fe72f 100644 --- a/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx @@ -57,9 +57,8 @@ export const MeasureDialog = React.memo(function MeasureDialog(props: MeasureDia .changeWithParts([SqlWithPart.simple('t', QuerySource.stripToBaseSource(querySource.query))]) .addSelect(L('Overall').as('label')) .addSelect(expression.as('value')) - .applyIf( - querySource.baseColumns.find(column => column.isTimeColumn()), - q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), ) .toString(); }, [querySource.query, formula]); diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss new file mode 100644 index 00000000000..ef7650c1f65 --- /dev/null +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../../../../variables'; + +.nested-column-dialog { + &.#{$bp-ns}-dialog { + width: 50vw; + min-height: 540px; + } + + .#{$bp-ns}-dialog-body { + display: flex; + flex-direction: column; + + .path-selector { + flex: 1; + padding: 5px 0; + height: 400px; + overflow: auto; + border-left: 1px solid rgba(15, 19, 32, 0.4); + border-right: 1px solid rgba(15, 19, 32, 0.4); + } + } + + .#{$bp-ns}-dialog-footer { + margin-top: 0; + } +} diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx new file mode 100644 index 00000000000..8ec09085f22 --- /dev/null +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Button, + ButtonGroup, + Classes, + Dialog, + FormGroup, + InputGroup, + Intent, + Menu, + Tag, +} from '@blueprintjs/core'; +import type { SqlExpression } from '@druid-toolkit/query'; +import { type QueryResult, F, sql, SqlFunction, SqlQuery } from '@druid-toolkit/query'; +import React, { useState } from 'react'; + +import { ClearableInput, Loader, MenuCheckbox } from '../../../../../components'; +import { useQueryManager } from '../../../../../hooks'; +import { caseInsensitiveContains, filterMap, pluralIfNeeded } from '../../../../../utils'; +import { ExpressionMeta, QuerySource } from '../../../models'; +import { toggle } from '../../../utils'; + +import './nested-column-dialog.scss'; + +const ARRAY_CONCAT_AGG_SIZE = 10000; + +export interface NestedColumnDialogProps { + nestedColumn: SqlExpression; + onApply(newQuery: SqlQuery): void; + querySource: QuerySource; + runSqlQuery(query: string | SqlQuery): Promise; + onClose(): void; +} + +export const NestedColumnDialog = React.memo(function NestedColumnDialog( + props: NestedColumnDialogProps, +) { + const { nestedColumn, onApply, querySource, runSqlQuery, onClose } = props; + const [searchString, setSearchString] = useState(''); + const [selectedPaths, setSelectedPaths] = useState([]); + const [namingScheme, setNamingScheme] = useState(`${nestedColumn.getFirstColumnName()}[%]`); + + const [pathsState] = useQueryManager({ + query: nestedColumn, + processQuery: async nestedColumn => { + const query = SqlQuery.from(QuerySource.stripToBaseSource(querySource.query)) + .addSelect( + SqlFunction.decorated('ARRAY_CONCAT_AGG', 'DISTINCT', [ + F('JSON_PATHS', nestedColumn), + ARRAY_CONCAT_AGG_SIZE, + ]), + ) + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + ); + + const pathResult = await runSqlQuery(query); + + const paths = pathResult.rows[0]?.[0]; + if (!Array.isArray(paths)) throw new Error('Could not get paths'); + + return paths; + }, + }); + + const paths = pathsState.data; + return ( + +
+

+ Replace {String(nestedColumn.getOutputName())} with path expansions for + the selected paths. +

+ {pathsState.isLoading() && } + {pathsState.getErrorMessage()} + {paths && ( + + + + {filterMap(paths, (path, i) => { + if (!caseInsensitiveContains(path, searchString)) return; + return ( + setSelectedPaths(toggle(selectedPaths, path))} + text={path} + /> + ); + })} + + +
+
+
+
+
+
+
+ + ); +}); diff --git a/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx b/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx index f6489eb17e2..2de75e32d86 100644 --- a/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx @@ -40,6 +40,7 @@ import type { Rename } from '../../utils'; import { ColumnDialog } from './column-dialog/column-dialog'; import { MeasureDialog } from './measure-dialog/measure-dialog'; +import { NestedColumnDialog } from './nested-column-dialog/nested-column-dialog'; import './resource-pane.scss'; @@ -67,6 +68,9 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) { const [columnSearch, setColumnSearch] = useState(''); const [columnEditorOpenOn, setColumnEditorOpenOn] = useState(); + const [nestedColumnEditorOpenOn, setNestedColumnEditorOpenOn] = useState< + SqlExpression | undefined + >(); const [measureEditorOpenOn, setMeasureEditorOpenOn] = useState(); function applyUtil(nameTransform: (columnName: string) => string) { @@ -112,6 +116,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
{filterMap(querySource.columns, (column, i) => { const columnName = column.name; + const isNestedColumn = column.nativeType === 'COMPLEX'; if (!caseInsensitiveContains(columnName, columnSearch)) return; return ( - {onFilter && ( + {isNestedColumn ? ( onFilter(column)} + icon={IconNames.EXPAND_ALL} + text="Expand nested column" + onClick={() => + setNestedColumnEditorOpenOn( + querySource.getSourceExpressionForColumn(columnName), + ) + } /> + ) : ( + <> + {onFilter && ( + onFilter(column)} + /> + )} + onShowColumn(column)} + /> + + )} - onShowColumn(column)} - /> -
{ e.dataTransfer.effectAllowed = 'all'; DragHelper.dragColumn = column; @@ -268,6 +287,15 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) { onClose={() => setColumnEditorOpenOn(undefined)} /> )} + {nestedColumnEditorOpenOn && ( + onQueryChange(newQuery, undefined)} + querySource={querySource} + runSqlQuery={runSqlQuery} + onClose={() => setNestedColumnEditorOpenOn(undefined)} + /> + )} {measureEditorOpenOn && ( MAX_PAST_QUERIES) QUERY_HISTORY.pop(); -} - -function getFormattedQueryHistory(): string { - return QUERY_HISTORY.map( - ({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`, - ).join('\n\n-----------------------------------------------------\n\n'); +function getStickyParameterValuesForModule(moduleId: string): ParameterValues { + return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {}; } // --------------------------------------- @@ -81,7 +74,7 @@ const queryRunner = new QueryRunner({ inflateDateStrategy: 'fromSqlTypes', executor: async (sqlQueryPayload, isSql, cancelToken) => { if (!isSql) throw new Error('should never get here'); - addQueryToHistory(sqlQueryPayload.query); + QUERY_LOG.addQuery(sqlQueryPayload.query); return Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken }); }, }); @@ -90,6 +83,9 @@ async function runSqlQuery(query: string | SqlQuery): Promise { try { return await queryRunner.runQuery({ query, + defaultQueryContext: { + sqlStringifyArrays: false, + }, }); } catch (e) { throw new DruidError(e); @@ -193,10 +189,25 @@ export const ExploreView = React.memo(function ExploreView() { } function resetParameterValues() { - setParameterValues({}); + setParameterValues(getStickyParameterValuesForModule(moduleId)); } function updateParameterValues(newParameterValues: ParameterValues) { + // Evaluate sticky-ness + if (module) { + const currentExploreSticky = localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY) || {}; + const currentModuleSticky = currentExploreSticky[moduleId] || {}; + const newModuleSticky = { + ...currentModuleSticky, + ...mapRecord(newParameterValues, (v, k) => (module.parameters[k]?.sticky ? v : undefined)), + }; + + localStorageSetJson(LocalStorageKeys.EXPLORE_STICKY, { + ...currentExploreSticky, + [moduleId]: isEmpty(newModuleSticky) ? undefined : newModuleSticky, + }); + } + setParameterValues({ ...parameterValues, ...newParameterValues }); } @@ -311,7 +322,8 @@ export const ExploreView = React.memo(function ExploreView() { ]} selectedModuleId={moduleId} onSelectedModuleIdChange={newModuleId => { - const newParameterValues: ParameterValues = {}; + const newParameterValues = getStickyParameterValuesForModule(newModuleId); + const oldModule = ModuleRepository.getModule(moduleId); const newModule = ModuleRepository.getModule(newModuleId); if (oldModule && newModule) { @@ -349,9 +361,9 @@ export const ExploreView = React.memo(function ExploreView() { { - copy(QUERY_HISTORY[0]?.sqlQuery, { format: 'text/plain' }); + copy(QUERY_LOG.getLastQuery()!, { format: 'text/plain' }); AppToaster.show({ message: `Copied query to clipboard`, intent: Intent.SUCCESS, @@ -360,9 +372,9 @@ export const ExploreView = React.memo(function ExploreView() { /> { - setShownText(getFormattedQueryHistory()); + setShownText(QUERY_LOG.getFormatted()); }} /> ': return [ diff --git a/web-console/src/views/explore-view/models/parameter.ts b/web-console/src/views/explore-view/models/parameter.ts index b7a952a1400..f4a5f622d73 100644 --- a/web-console/src/views/explore-view/models/parameter.ts +++ b/web-console/src/views/explore-view/models/parameter.ts @@ -85,6 +85,7 @@ export type TypedParameterDefinition = TypedE | ParameterTypes[Type] | ((querySource: QuerySource) => ParameterTypes[Type] | undefined); + sticky?: boolean; required?: ModuleFunctor; description?: ModuleFunctor; placeholder?: string; diff --git a/web-console/src/views/explore-view/models/query-source.ts b/web-console/src/views/explore-view/models/query-source.ts index a8a5257e311..4ff83cf14d6 100644 --- a/web-console/src/views/explore-view/models/query-source.ts +++ b/web-console/src/views/explore-view/models/query-source.ts @@ -95,7 +95,9 @@ export class QuerySource { let effectiveColumns = columns; if (query.getSelectExpressionsArray().some(ex => ex instanceof SqlStar)) { // The query has a star so carefully pick the columns that make sense - effectiveColumns = columns.filter(c => c.sqlType !== 'OTHER'); + effectiveColumns = columns.filter( + c => c.sqlType !== 'OTHER' || c.nativeType === 'COMPLEX', + ); } let measures = Measure.extractQueryMeasures(query); @@ -179,6 +181,10 @@ export class QuerySource { return this.measures.some(m => m.name === name); } + public hasBaseTimeColumn(): boolean { + return this.baseColumns.some(column => column.isTimeColumn()); + } + public getSourceExpressionForColumn(outputName: string): SqlExpression { const selectExpressionsArray = this.query.getSelectExpressionsArray(); @@ -224,12 +230,12 @@ export class QuerySource { return noStarQuery.addSelect(newExpression); } - public addColumnAfter(neighborName: string, newExpression: SqlExpression): SqlQuery { + public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery { const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); return noStarQuery.changeSelectExpressions( noStarQuery .getSelectExpressionsArray() - .flatMap(ex => (ex.getOutputName() === neighborName ? [ex, newExpression] : ex)), + .flatMap(ex => (ex.getOutputName() === neighborName ? [ex, ...newExpressions] : ex)), ); } diff --git a/web-console/src/views/explore-view/modules/grouping-table-module.tsx b/web-console/src/views/explore-view/modules/grouping-table-module.tsx index c8aa74922ec..e2cba7cef6d 100644 --- a/web-console/src/views/explore-view/modules/grouping-table-module.tsx +++ b/web-console/src/views/explore-view/modules/grouping-table-module.tsx @@ -105,6 +105,7 @@ ModuleRepository.registerModule({ count: `Show ' values'`, }, defaultValue: 'null', + sticky: true, visible: ({ parameterValues }) => Boolean((parameterValues.showColumns || []).length), }, pivotColumn: { diff --git a/web-console/src/views/explore-view/modules/record-table-module.tsx b/web-console/src/views/explore-view/modules/record-table-module.tsx index b272a4dfabe..38e2cfb6fac 100644 --- a/web-console/src/views/explore-view/modules/record-table-module.tsx +++ b/web-console/src/views/explore-view/modules/record-table-module.tsx @@ -21,10 +21,9 @@ import React, { useMemo } from 'react'; import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; -import { - calculateInitPageSize, - GenericOutputTable, -} from '../components/generic-output-table/generic-output-table'; +import type { ColumnHint } from '../../../utils'; +import { filterMap } from '../../../utils'; +import { calculateInitPageSize, GenericOutputTable } from '../components'; import { ModuleRepository } from '../module-repository/module-repository'; import './record-table-module.scss'; @@ -33,6 +32,7 @@ interface RecordTableParameterValues { maxRows: number; ascending: boolean; showTypeIcons: boolean; + hideNullColumns: boolean; } ModuleRepository.registerModule({ @@ -50,10 +50,18 @@ ModuleRepository.registerModule({ ascending: { type: 'boolean', defaultValue: false, + sticky: true, }, showTypeIcons: { type: 'boolean', defaultValue: true, + sticky: true, + }, + hideNullColumns: { + type: 'boolean', + label: 'Hide all null columns', + defaultValue: false, + sticky: true, }, }, component: function RecordTableModule(props) { @@ -77,6 +85,18 @@ ModuleRepository.registerModule({ }); const resultData = resultState.getSomeData(); + + let columnHints: Map | undefined; + if (parameterValues.hideNullColumns && resultData) { + columnHints = new Map( + filterMap(resultData.header, (column, i) => + resultData.getColumnByIndex(i)?.every(v => v == null) + ? [column.name, { hidden: true }] + : undefined, + ), + ); + } + return (
{resultState.error ? ( @@ -84,6 +104,7 @@ ModuleRepository.registerModule({ ) : resultData ? ( ({ }, snappyHighlight: { type: 'boolean', - label: 'Snap highlight to nearest dates', + label: 'Snap highlight to granularity', defaultValue: true, + sticky: true, }, }, component: function TimeChartModule(props) { diff --git a/web-console/src/views/explore-view/query-macros/aggregate.ts b/web-console/src/views/explore-view/query-macros/aggregate.ts index 9a9b9211eb0..377c7f5b02d 100644 --- a/web-console/src/views/explore-view/query-macros/aggregate.ts +++ b/web-console/src/views/explore-view/query-macros/aggregate.ts @@ -20,6 +20,7 @@ import { C, SqlFunction, SqlQuery } from '@druid-toolkit/query'; import { filterMap, uniq } from '../../../utils'; import { Measure } from '../models'; +import { KNOWN_AGGREGATIONS } from '../utils'; export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery { const usedMeasures: Map = new Map(); @@ -35,7 +36,14 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery if (!measure) throw new Error(`${Measure.AGGREGATE} of unknown measure '${measureName}'`); usedMeasures.set(measureName, true); - return measure.expression; + + let measureExpression = measure.expression; + const filter = ex.getWhereExpression(); + if (filter) { + measureExpression = measureExpression.addFilterToAggregations(filter, KNOWN_AGGREGATIONS); + } + + return measureExpression; } // If we encounter a (the) query with the measure definitions, and we have used those measures then expand out all the columns within them diff --git a/web-console/src/views/explore-view/utils/index.ts b/web-console/src/views/explore-view/utils/index.ts index 215b4c9d44d..8469c83b927 100644 --- a/web-console/src/views/explore-view/utils/index.ts +++ b/web-console/src/views/explore-view/utils/index.ts @@ -21,8 +21,10 @@ export * from './duration'; export * from './filter-pattern-helpers'; export * from './general'; export * from './get-auto-granularity'; +export * from './known-aggregations'; export * from './max-time-for-table'; export * from './misc'; +export * from './query-log'; export * from './snap-to-granularity'; export * from './table-query'; export * from './time-manipulation'; diff --git a/web-console/src/views/explore-view/utils/known-aggregations.ts b/web-console/src/views/explore-view/utils/known-aggregations.ts new file mode 100644 index 00000000000..0df2420648c --- /dev/null +++ b/web-console/src/views/explore-view/utils/known-aggregations.ts @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const KNOWN_AGGREGATIONS = [ + 'COUNT', + 'SUM', + 'MIN', + 'MAX', + 'AVG', + 'APPROX_COUNT_DISTINCT', + 'APPROX_COUNT_DISTINCT_BUILTIN', + 'APPROX_QUANTILE', + 'APPROX_QUANTILE_FIXED_BUCKETS', + 'BLOOM_FILTER', + 'VAR_POP', + 'VAR_SAMP', + 'VARIANCE', + 'STDDEV_POP', + 'STDDEV_SAMP', + 'STDDEV', + 'EARLIEST', + 'EARLIEST_BY', + 'LATEST', + 'LATEST_BY', + 'ANY_VALUE', + 'GROUPING', + 'ARRAY_AGG', + 'ARRAY_AGG', + 'ARRAY_CONCAT_AGG', + 'ARRAY_CONCAT_AGG', + 'STRING_AGG', + 'LISTAGG', + 'BIT_AND', + 'BIT_OR', + 'BIT_XOR', + 'APPROX_COUNT_DISTINCT_DS_THETA', + 'DS_THETA', + 'APPROX_QUANTILE_DS', + 'DS_QUANTILES_SKETCH', + 'DS_TUPLE_DOUBLES', + 'DS_TUPLE_DOUBLES', + 'TDIGEST_QUANTILE', + 'TDIGEST_GENERATE_SKETCH', +]; diff --git a/web-console/src/views/explore-view/utils/query-log.ts b/web-console/src/views/explore-view/utils/query-log.ts new file mode 100644 index 00000000000..20bfb6ef0d4 --- /dev/null +++ b/web-console/src/views/explore-view/utils/query-log.ts @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface QueryLogEntry { + time: Date; + sqlQuery: string; +} + +const MAX_QUERIES_TO_LOG = 10; + +export class QueryLog { + private readonly queryLog: QueryLogEntry[] = []; + + public length(): number { + return this.queryLog.length; + } + + public addQuery(sqlQuery: string): void { + const { queryLog } = this; + queryLog.unshift({ time: new Date(), sqlQuery }); + while (queryLog.length > MAX_QUERIES_TO_LOG) queryLog.pop(); + } + + public getLastQuery(): string | undefined { + return this.queryLog[0]?.sqlQuery; + } + + public getFormatted(): string { + return this.queryLog + .map(({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`) + .join('\n\n-----------------------------------------------------\n\n'); + } +} diff --git a/web-console/src/views/explore-view/utils/table-query.ts b/web-console/src/views/explore-view/utils/table-query.ts index 136b5644975..e7f1499e5f2 100644 --- a/web-console/src/views/explore-view/utils/table-query.ts +++ b/web-console/src/views/explore-view/utils/table-query.ts @@ -37,6 +37,7 @@ import { Measure } from '../models'; import { formatDuration } from './duration'; import { addTableScope } from './general'; +import { KNOWN_AGGREGATIONS } from './known-aggregations'; import type { Compare } from './time-manipulation'; import { computeWhereForCompares } from './time-manipulation'; @@ -48,35 +49,6 @@ export type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPerce export type RestrictTop = 'always' | 'never'; -const KNOWN_AGGREGATIONS = [ - 'COUNT', - 'SUM', - 'MIN', - 'MAX', - 'AVG', - 'APPROX_COUNT_DISTINCT', - 'APPROX_COUNT_DISTINCT_DS_HLL', - 'APPROX_COUNT_DISTINCT_DS_THETA', - 'DS_HLL', - 'DS_THETA', - 'APPROX_QUANTILE', - 'APPROX_QUANTILE_DS', - 'APPROX_QUANTILE_FIXED_BUCKETS', - 'DS_QUANTILES_SKETCH', - 'BLOOM_FILTER', - 'TDIGEST_QUANTILE', - 'TDIGEST_GENERATE_SKETCH', - 'VAR_POP', - 'VAR_SAMP', - 'VARIANCE', - 'STDDEV_POP', - 'STDDEV_SAMP', - 'STDDEV', - 'EARLIEST', - 'LATEST', - 'ANY_VALUE', -]; - const DRUID_DEFAULT_TOTAL_SUB_QUERY_LIMIT = 100000; const COMMON_NAME = 'common';