Web console: misc fixes to the Explore view (#17213)

* make record table able to hide column

* stickyness

* refactor query log

* fix measure drag

* start nested column dialog

* nested expand

* fix filtering on Measures

* use output name

* fix scrolling

* select all / none

* use ARRAY_CONCAT_AGG

* no need to limit if aggregating

* remove magic number

* better search

* update arg list

* add, don't replace

web-console/src/utils/local-storage-keys.tsx

@@ -58,6 +58,7 @@ export const LocalStorageKeys = {
   SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const,
 
   EXPLORE_STATE: 'explore-state' as const,
+  EXPLORE_STICKY: 'explore-sticky' as const,
 };
 export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys];
 

web-console/src/utils/table-helpers.ts

@@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination {
 export interface ColumnHint {
   displayName?: string;
   group?: string;
+  hidden?: boolean;
   expressionForWhere?: SqlExpression;
   formatter?: (x: any) => string;
 }

web-console/src/views/explore-view/components/control-pane/control-pane.tsx

@@ -194,7 +194,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
         };
         return {
           element: (
-            <NamedExpressionsInput
+            <NamedExpressionsInput<ExpressionMeta>
               allowReordering
               values={effectiveValue ? [effectiveValue] : []}
               onValuesChange={vs => onValueChange(vs[0])}
@@ -223,7 +223,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
             );
         return {
           element: (
-            <NamedExpressionsInput
+            <NamedExpressionsInput<ExpressionMeta>
               allowReordering
               values={effectiveValue as ExpressionMeta[]}
               onValuesChange={onValueChange}
@@ -266,7 +266,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
       case 'measure': {
         return {
           element: (
-            <NamedExpressionsInput
+            <NamedExpressionsInput<Measure>
               values={effectiveValue ? [effectiveValue] : []}
               onValuesChange={vs => onValueChange(vs[0])}
               singleton
@@ -284,9 +284,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
             />
           ),
           onDropColumn: column => {
-            const measures = Measure.getPossibleMeasuresForColumn(column);
+            const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
-            if (!measures.length) return;
+              p => !effectiveValue || effectiveValue.name !== p.name,
-            onValueChange(measures[0]);
+            );
+            if (!candidateMeasures.length) return;
+            onValueChange(candidateMeasures[0]);
           },
           onDropMeasure: onValueChange,
         };
@@ -313,11 +315,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) {
             />
           ),
           onDropColumn: column => {
-            const measures = Measure.getPossibleMeasuresForColumn(column).filter(
+            const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter(
-              p => !effectiveValue.some((v: ExpressionMeta) => v.name === p.name),
+              p => !effectiveValue.some((v: Measure) => v.name === p.name),
             );
-            if (!measures.length) return;
+            if (!candidateMeasures.length) return;
-            onValueChange(effectiveValue.concat(measures[0]));
+            onValueChange(effectiveValue.concat(candidateMeasures[0]));
           },
           onDropMeasure: measure => {
             onValueChange(effectiveValue.concat(measure));

web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx

@@ -61,6 +61,7 @@ export const NamedExpressionsInput = function NamedExpressionsInput<
 
   const onDragOver = useCallback(
     (e: React.DragEvent, i: number) => {
+      if (dragIndex === -1) return;
       const targetRect = e.currentTarget.getBoundingClientRect();
       const before = e.clientX - targetRect.left <= targetRect.width / 2;
       setDropBefore(before);

web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx

@@ -52,6 +52,7 @@ export const ContainsFilterControl = React.memo(function ContainsFilterControl(
           ),
         )
         .changeOrderByExpression(F.count().toOrderByExpression('DESC'))
+        .changeLimitValue(101)
         .toString(),
     // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
     [querySource.query, filter, column, contains, negated],

web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx

@@ -58,6 +58,7 @@ export const RegexpFilterControl = React.memo(function RegexpFilterControl(
           SqlExpression.and(filter, regexp ? filterPatternToExpression(filterPattern) : undefined),
         )
         .changeOrderByExpression(F.count().toOrderByExpression('DESC'))
+        .changeLimitValue(101)
         .toString(),
     // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps
     [querySource.query, filter, column, regexp, negated],

web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx

@@ -16,14 +16,15 @@
  * limitations under the License.
  */
 
-import { FormGroup, InputGroup, Menu, MenuItem } from '@blueprintjs/core';
+import { FormGroup, Menu, MenuItem } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
-import type { QueryResult, SqlQuery, ValuesFilterPattern } from '@druid-toolkit/query';
+import type { QueryResult, ValuesFilterPattern } from '@druid-toolkit/query';
-import { C, F, L, SqlExpression, SqlLiteral } from '@druid-toolkit/query';
+import { C, F, SqlExpression, SqlQuery } from '@druid-toolkit/query';
 import React, { useMemo, useState } from 'react';
 
+import { ClearableInput } from '../../../../../../components';
 import { useQueryManager } from '../../../../../../hooks';
-import { caseInsensitiveContains } from '../../../../../../utils';
+import { caseInsensitiveContains, filterMap } from '../../../../../../utils';
 import type { QuerySource } from '../../../../models';
 import { toggle } from '../../../../utils';
 import { ColumnValue } from '../../column-value/column-value';
@@ -46,21 +47,21 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
   const [initValues] = useState(selectedValues);
   const [searchString, setSearchString] = useState('');
 
-  const valuesQuery = useMemo(() => {
+  const valuesQuery = useMemo(
-    const columnRef = C(column);
+    () =>
-    const queryParts: string[] = [`SELECT ${columnRef.as('c')}`, `FROM (${querySource.query})`];
+      SqlQuery.from(querySource.query)
-
+        .addSelect(C(column).as('c'), { addToGroupBy: 'end' })
-    const filterEx = SqlExpression.and(
+        .changeWhereExpression(
-      filter,
+          SqlExpression.and(
-      searchString ? F('ICONTAINS_STRING', columnRef, L(searchString)) : undefined,
+            filter,
-    );
+            searchString ? F('ICONTAINS_STRING', C(column), searchString) : undefined,
-    if (!(filterEx instanceof SqlLiteral)) {
+          ),
-      queryParts.push(`WHERE ${filterEx}`);
+        )
-    }
+        .changeOrderByExpression(F.count().toOrderByExpression('DESC'))
-
+        .changeLimitValue(101)
-    queryParts.push(`GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 101`);
+        .toString(),
-    return queryParts.join('\n');
+    [querySource.query, filter, column, searchString],
-  }, [querySource.query, filter, column, searchString]);
+  );
 
   const [valuesState] = useQueryManager<string, any[]>({
     query: valuesQuery,
@@ -77,42 +78,37 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl(
   if (values) {
     valuesToShow = valuesToShow.concat(values.filter(v => !initValues.includes(v)));
   }
-  if (searchString) {
-    valuesToShow = valuesToShow.filter(v => caseInsensitiveContains(v, searchString));
-  }
 
   const showSearch = querySource.columns.find(c => c.name === column)?.sqlType !== 'BOOLEAN';
-
   return (
     <FormGroup className="values-filter-control">
       {showSearch && (
-        <InputGroup
+        <ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
-          value={searchString}
-          onChange={e => setSearchString(e.target.value)}
-          placeholder="Search"
-        />
       )}
       <Menu className="value-list">
-        {valuesToShow.map((v, i) => (
+        {filterMap(valuesToShow, (v, i) => {
-          <MenuItem
+          if (!caseInsensitiveContains(v, searchString)) return;
-            key={i}
+          return (
-            icon={
+            <MenuItem
-              selectedValues.includes(v)
+              key={i}
-                ? negated
+              icon={
-                  ? IconNames.DELETE
+                selectedValues.includes(v)
-                  : IconNames.TICK_CIRCLE
+                  ? negated
-                : IconNames.CIRCLE
+                    ? IconNames.DELETE
-            }
+                    : IconNames.TICK_CIRCLE
-            text={<ColumnValue value={v} />}
+                  : IconNames.CIRCLE
-            shouldDismissPopover={false}
+              }
-            onClick={e => {
+              text={<ColumnValue value={v} />}
-              setFilterPattern({
+              shouldDismissPopover={false}
-                ...filterPattern,
+              onClick={e => {
-                values: e.altKey ? [v] : toggle(selectedValues, v),
+                setFilterPattern({
-              });
+                  ...filterPattern,
-            }}
+                  values: e.altKey ? [v] : toggle(selectedValues, v),
-          />
+                });
-        ))}
+              }}
+            />
+          );
+        })}
         {valuesState.loading && <MenuItem icon={IconNames.BLANK} text="Loading..." disabled />}
       </Menu>
     </FormGroup>

web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx

@@ -428,6 +428,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
           columns={columnNester(
             queryResult.header.map((column, i) => {
               const h = column.name;
+              const hint = columnHints?.get(h);
               const icon = showTypeIcons ? columnToIcon(column) : undefined;
 
               return {
@@ -446,9 +447,10 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
                 },
                 headerClassName: getHeaderClassName(h),
                 accessor: String(i),
+                show: !hint?.hidden,
                 Cell(row) {
                   const value = row.value;
-                  const formatter = columnHints?.get(h)?.formatter || formatNumber;
+                  const formatter = hint?.formatter || formatNumber;
                   return (
                     <div>
                       <Popover content={<Deferred content={() => getCellMenu(column, i, value)} />}>

web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx

@@ -48,9 +48,8 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
     if (!expression) return;
     return SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
       .addSelect(F.cast(expression, 'VARCHAR').as('v'), { addToGroupBy: 'end' })
-      .applyIf(
+      .applyIf(querySource.hasBaseTimeColumn(), q =>
-        querySource.baseColumns.find(column => column.isTimeColumn()),
+        q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
-        q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
       )
       .changeLimitValue(100)
       .toString();
@@ -151,7 +150,7 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog
                   } else {
                     onApply(
                       querySource.changeColumn(initExpressionName, newExpression),
-                      new Map([[initExpression.getOutputName()!, newExpression.getOutputName()!]]),
+                      new Map([[initExpressionName, newExpression.getOutputName()!]]),
                     );
                   }
                 } else {

web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx

@@ -57,9 +57,8 @@ export const MeasureDialog = React.memo(function MeasureDialog(props: MeasureDia
       .changeWithParts([SqlWithPart.simple('t', QuerySource.stripToBaseSource(querySource.query))])
       .addSelect(L('Overall').as('label'))
       .addSelect(expression.as('value'))
-      .applyIf(
+      .applyIf(querySource.hasBaseTimeColumn(), q =>
-        querySource.baseColumns.find(column => column.isTimeColumn()),
+        q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
-        q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
       )
       .toString();
   }, [querySource.query, formula]);

web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../../../../variables';
+
+.nested-column-dialog {
+  &.#{$bp-ns}-dialog {
+    width: 50vw;
+    min-height: 540px;
+  }
+
+  .#{$bp-ns}-dialog-body {
+    display: flex;
+    flex-direction: column;
+
+    .path-selector {
+      flex: 1;
+      padding: 5px 0;
+      height: 400px;
+      overflow: auto;
+      border-left: 1px solid rgba(15, 19, 32, 0.4);
+      border-right: 1px solid rgba(15, 19, 32, 0.4);
+    }
+  }
+
+  .#{$bp-ns}-dialog-footer {
+    margin-top: 0;
+  }
+}

web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx

@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Button,
+  ButtonGroup,
+  Classes,
+  Dialog,
+  FormGroup,
+  InputGroup,
+  Intent,
+  Menu,
+  Tag,
+} from '@blueprintjs/core';
+import type { SqlExpression } from '@druid-toolkit/query';
+import { type QueryResult, F, sql, SqlFunction, SqlQuery } from '@druid-toolkit/query';
+import React, { useState } from 'react';
+
+import { ClearableInput, Loader, MenuCheckbox } from '../../../../../components';
+import { useQueryManager } from '../../../../../hooks';
+import { caseInsensitiveContains, filterMap, pluralIfNeeded } from '../../../../../utils';
+import { ExpressionMeta, QuerySource } from '../../../models';
+import { toggle } from '../../../utils';
+
+import './nested-column-dialog.scss';
+
+const ARRAY_CONCAT_AGG_SIZE = 10000;
+
+export interface NestedColumnDialogProps {
+  nestedColumn: SqlExpression;
+  onApply(newQuery: SqlQuery): void;
+  querySource: QuerySource;
+  runSqlQuery(query: string | SqlQuery): Promise<QueryResult>;
+  onClose(): void;
+}
+
+export const NestedColumnDialog = React.memo(function NestedColumnDialog(
+  props: NestedColumnDialogProps,
+) {
+  const { nestedColumn, onApply, querySource, runSqlQuery, onClose } = props;
+  const [searchString, setSearchString] = useState('');
+  const [selectedPaths, setSelectedPaths] = useState<string[]>([]);
+  const [namingScheme, setNamingScheme] = useState(`${nestedColumn.getFirstColumnName()}[%]`);
+
+  const [pathsState] = useQueryManager({
+    query: nestedColumn,
+    processQuery: async nestedColumn => {
+      const query = SqlQuery.from(QuerySource.stripToBaseSource(querySource.query))
+        .addSelect(
+          SqlFunction.decorated('ARRAY_CONCAT_AGG', 'DISTINCT', [
+            F('JSON_PATHS', nestedColumn),
+            ARRAY_CONCAT_AGG_SIZE,
+          ]),
+        )
+        .applyIf(querySource.hasBaseTimeColumn(), q =>
+          q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`),
+        );
+
+      const pathResult = await runSqlQuery(query);
+
+      const paths = pathResult.rows[0]?.[0];
+      if (!Array.isArray(paths)) throw new Error('Could not get paths');
+
+      return paths;
+    },
+  });
+
+  const paths = pathsState.data;
+  return (
+    <Dialog className="nested-column-dialog" isOpen onClose={onClose} title="Expand nested column">
+      <div className={Classes.DIALOG_BODY}>
+        <p>
+          Replace <Tag minimal>{String(nestedColumn.getOutputName())}</Tag> with path expansions for
+          the selected paths.
+        </p>
+        {pathsState.isLoading() && <Loader />}
+        {pathsState.getErrorMessage()}
+        {paths && (
+          <FormGroup>
+            <ClearableInput value={searchString} onChange={setSearchString} placeholder="Search" />
+            <Menu className="path-selector">
+              {filterMap(paths, (path, i) => {
+                if (!caseInsensitiveContains(path, searchString)) return;
+                return (
+                  <MenuCheckbox
+                    key={i}
+                    checked={selectedPaths.includes(path)}
+                    onChange={() => setSelectedPaths(toggle(selectedPaths, path))}
+                    text={path}
+                  />
+                );
+              })}
+            </Menu>
+            <ButtonGroup fill>
+              <Button
+                text="Select all"
+                onClick={() =>
+                  // Select all paths matching the shown search string
+                  setSelectedPaths(
+                    paths.filter(path => caseInsensitiveContains(path, searchString)),
+                  )
+                }
+              />
+              <Button
+                text="Select none"
+                onClick={() =>
+                  // Remove from selection all the paths matching the search string
+                  setSelectedPaths(
+                    selectedPaths.filter(path => !caseInsensitiveContains(path, searchString)),
+                  )
+                }
+              />
+            </ButtonGroup>
+          </FormGroup>
+        )}
+        <FormGroup label="Nameing scheme">
+          <InputGroup
+            value={namingScheme}
+            onChange={e => {
+              setNamingScheme(e.target.value.slice(0, ExpressionMeta.MAX_NAME_LENGTH));
+            }}
+          />
+        </FormGroup>
+      </div>
+      <div className={Classes.DIALOG_FOOTER}>
+        <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+          <div className="edit-column-dialog-buttons">
+            <Button text="Cancel" onClick={onClose} />
+            <Button
+              text={
+                selectedPaths.length
+                  ? `Add ${pluralIfNeeded(selectedPaths.length, 'column')}`
+                  : 'Select path'
+              }
+              disabled={!selectedPaths.length}
+              intent={Intent.PRIMARY}
+              onClick={() => {
+                const effectiveNamingScheme = namingScheme.includes('%')
+                  ? namingScheme
+                  : namingScheme + '[%]';
+                onApply(
+                  querySource.addColumnAfter(
+                    nestedColumn.getOutputName()!,
+                    ...selectedPaths.map(path =>
+                      F('JSON_VALUE', nestedColumn, path).as(
+                        querySource.getAvailableName(
+                          effectiveNamingScheme.replaceAll('%', path.replace(/^\$\./, '')),
+                        ),
+                      ),
+                    ),
+                  ),
+                );
+                onClose();
+              }}
+            />
+          </div>
+        </div>
+      </div>
+    </Dialog>
+  );
+});

web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx

@@ -40,6 +40,7 @@ import type { Rename } from '../../utils';
 
 import { ColumnDialog } from './column-dialog/column-dialog';
 import { MeasureDialog } from './measure-dialog/measure-dialog';
+import { NestedColumnDialog } from './nested-column-dialog/nested-column-dialog';
 
 import './resource-pane.scss';
 
@@ -67,6 +68,9 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
   const [columnSearch, setColumnSearch] = useState('');
 
   const [columnEditorOpenOn, setColumnEditorOpenOn] = useState<ColumnEditorOpenOn | undefined>();
+  const [nestedColumnEditorOpenOn, setNestedColumnEditorOpenOn] = useState<
+    SqlExpression | undefined
+  >();
   const [measureEditorOpenOn, setMeasureEditorOpenOn] = useState<MeasureEditorOpenOn | undefined>();
 
   function applyUtil(nameTransform: (columnName: string) => string) {
@@ -112,6 +116,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
       <div className="column-resource-list">
         {filterMap(querySource.columns, (column, i) => {
           const columnName = column.name;
+          const isNestedColumn = column.nativeType === 'COMPLEX<json>';
           if (!caseInsensitiveContains(columnName, columnSearch)) return;
           return (
             <Popover
@@ -120,19 +125,33 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
               position="right"
               content={
                 <Menu>
-                  {onFilter && (
+                  {isNestedColumn ? (
                     <MenuItem
-                      icon={IconNames.FILTER}
+                      icon={IconNames.EXPAND_ALL}
-                      text="Filter"
+                      text="Expand nested column"
-                      onClick={() => onFilter(column)}
+                      onClick={() =>
+                        setNestedColumnEditorOpenOn(
+                          querySource.getSourceExpressionForColumn(columnName),
+                        )
+                      }
                     />
+                  ) : (
+                    <>
+                      {onFilter && (
+                        <MenuItem
+                          icon={IconNames.FILTER}
+                          text="Filter"
+                          onClick={() => onFilter(column)}
+                        />
+                      )}
+                      <MenuItem
+                        icon={IconNames.EYE_OPEN}
+                        text="Show"
+                        onClick={() => onShowColumn(column)}
+                      />
+                      <MenuDivider />
+                    </>
                   )}
-                  <MenuItem
-                    icon={IconNames.EYE_OPEN}
-                    text="Show"
-                    onClick={() => onShowColumn(column)}
-                  />
-                  <MenuDivider />
                   <MenuItem
                     icon={IconNames.EDIT}
                     text="Edit"
@@ -165,7 +184,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
             >
               <div
                 className={Classes.MENU_ITEM}
-                draggable
+                draggable={!isNestedColumn}
                 onDragStart={e => {
                   e.dataTransfer.effectAllowed = 'all';
                   DragHelper.dragColumn = column;
@@ -268,6 +287,15 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
           onClose={() => setColumnEditorOpenOn(undefined)}
         />
       )}
+      {nestedColumnEditorOpenOn && (
+        <NestedColumnDialog
+          nestedColumn={nestedColumnEditorOpenOn}
+          onApply={newQuery => onQueryChange(newQuery, undefined)}
+          querySource={querySource}
+          runSqlQuery={runSqlQuery}
+          onClose={() => setNestedColumnEditorOpenOn(undefined)}
+        />
+      )}
       {measureEditorOpenOn && (
         <MeasureDialog
           initMeasure={measureEditorOpenOn.measure}

web-console/src/views/explore-view/explore-view.tsx

@@ -30,7 +30,15 @@ import { useStore } from 'zustand';
 import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
 import { useHashAndLocalStorageHybridState, useQueryManager } from '../../hooks';
 import { Api, AppToaster } from '../../singletons';
-import { DruidError, LocalStorageKeys, queryDruidSql } from '../../utils';
+import {
+  DruidError,
+  isEmpty,
+  localStorageGetJson,
+  LocalStorageKeys,
+  localStorageSetJson,
+  mapRecord,
+  queryDruidSql,
+} from '../../utils';
 
 import {
   ControlPane,
@@ -50,29 +58,14 @@ import { QuerySource } from './models';
 import { ModuleRepository } from './module-repository/module-repository';
 import { rewriteAggregate, rewriteMaxDataTime } from './query-macros';
 import type { Rename } from './utils';
-import { adjustTransferValue, normalizeType } from './utils';
+import { adjustTransferValue, normalizeType, QueryLog } from './utils';
 
 import './explore-view.scss';
 
-// ---------------------------------------
+const QUERY_LOG = new QueryLog();
 
-interface QueryHistoryEntry {
+function getStickyParameterValuesForModule(moduleId: string): ParameterValues {
-  time: Date;
+  return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {};
-  sqlQuery: string;
-}
-
-const MAX_PAST_QUERIES = 10;
-const QUERY_HISTORY: QueryHistoryEntry[] = [];
-
-function addQueryToHistory(sqlQuery: string): void {
-  QUERY_HISTORY.unshift({ time: new Date(), sqlQuery });
-  while (QUERY_HISTORY.length > MAX_PAST_QUERIES) QUERY_HISTORY.pop();
-}
-
-function getFormattedQueryHistory(): string {
-  return QUERY_HISTORY.map(
-    ({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`,
-  ).join('\n\n-----------------------------------------------------\n\n');
 }
 
 // ---------------------------------------
@@ -81,7 +74,7 @@ const queryRunner = new QueryRunner({
   inflateDateStrategy: 'fromSqlTypes',
   executor: async (sqlQueryPayload, isSql, cancelToken) => {
     if (!isSql) throw new Error('should never get here');
-    addQueryToHistory(sqlQueryPayload.query);
+    QUERY_LOG.addQuery(sqlQueryPayload.query);
     return Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken });
   },
 });
@@ -90,6 +83,9 @@ async function runSqlQuery(query: string | SqlQuery): Promise<QueryResult> {
   try {
     return await queryRunner.runQuery({
       query,
+      defaultQueryContext: {
+        sqlStringifyArrays: false,
+      },
     });
   } catch (e) {
     throw new DruidError(e);
@@ -193,10 +189,25 @@ export const ExploreView = React.memo(function ExploreView() {
   }
 
   function resetParameterValues() {
-    setParameterValues({});
+    setParameterValues(getStickyParameterValuesForModule(moduleId));
   }
 
   function updateParameterValues(newParameterValues: ParameterValues) {
+    // Evaluate sticky-ness
+    if (module) {
+      const currentExploreSticky = localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY) || {};
+      const currentModuleSticky = currentExploreSticky[moduleId] || {};
+      const newModuleSticky = {
+        ...currentModuleSticky,
+        ...mapRecord(newParameterValues, (v, k) => (module.parameters[k]?.sticky ? v : undefined)),
+      };
+
+      localStorageSetJson(LocalStorageKeys.EXPLORE_STICKY, {
+        ...currentExploreSticky,
+        [moduleId]: isEmpty(newModuleSticky) ? undefined : newModuleSticky,
+      });
+    }
+
     setParameterValues({ ...parameterValues, ...newParameterValues });
   }
 
@@ -311,7 +322,8 @@ export const ExploreView = React.memo(function ExploreView() {
             ]}
             selectedModuleId={moduleId}
             onSelectedModuleIdChange={newModuleId => {
-              const newParameterValues: ParameterValues = {};
+              const newParameterValues = getStickyParameterValuesForModule(newModuleId);
+
               const oldModule = ModuleRepository.getModule(moduleId);
               const newModule = ModuleRepository.getModule(newModuleId);
               if (oldModule && newModule) {
@@ -349,9 +361,9 @@ export const ExploreView = React.memo(function ExploreView() {
                 <MenuItem
                   icon={IconNames.DUPLICATE}
                   text="Copy last query"
-                  disabled={!QUERY_HISTORY.length}
+                  disabled={!QUERY_LOG.length()}
                   onClick={() => {
-                    copy(QUERY_HISTORY[0]?.sqlQuery, { format: 'text/plain' });
+                    copy(QUERY_LOG.getLastQuery()!, { format: 'text/plain' });
                     AppToaster.show({
                       message: `Copied query to clipboard`,
                       intent: Intent.SUCCESS,
@@ -360,9 +372,9 @@ export const ExploreView = React.memo(function ExploreView() {
                 />
                 <MenuItem
                   icon={IconNames.HISTORY}
-                  text="Show query history"
+                  text="Show query log"
                   onClick={() => {
-                    setShownText(getFormattedQueryHistory());
+                    setShownText(QUERY_LOG.getFormatted());
                   }}
                 />
                 <MenuItem

web-console/src/views/explore-view/models/measure.ts

@@ -90,7 +90,7 @@ export class Measure extends ExpressionMeta {
     }
 
     switch (column.nativeType) {
-      case 'BIGINT':
+      case 'LONG':
       case 'FLOAT':
       case 'DOUBLE':
         return [
@@ -103,16 +103,16 @@ export class Measure extends ExpressionMeta {
           new Measure({
             expression: F.min(C(column.name)),
           }),
-          new Measure({
-            expression: SqlFunction.countDistinct(C(column.name)),
-          }),
           new Measure({
             as: `P98 ${column.name}`,
             expression: F('APPROX_QUANTILE_DS', C(column.name), 0.98),
           }),
+          new Measure({
+            expression: SqlFunction.countDistinct(C(column.name)),
+          }),
         ];
 
-      case 'VARCHAR':
+      case 'STRING':
       case 'COMPLEX':
       case 'COMPLEX<hyperUnique>':
         return [

web-console/src/views/explore-view/models/parameter.ts

@@ -85,6 +85,7 @@ export type TypedParameterDefinition<Type extends keyof ParameterTypes> = TypedE
     | ParameterTypes[Type]
     | ((querySource: QuerySource) => ParameterTypes[Type] | undefined);
 
+  sticky?: boolean;
   required?: ModuleFunctor<boolean>;
   description?: ModuleFunctor<string>;
   placeholder?: string;

web-console/src/views/explore-view/models/query-source.ts

@@ -95,7 +95,9 @@ export class QuerySource {
     let effectiveColumns = columns;
     if (query.getSelectExpressionsArray().some(ex => ex instanceof SqlStar)) {
       // The query has a star so carefully pick the columns that make sense
-      effectiveColumns = columns.filter(c => c.sqlType !== 'OTHER');
+      effectiveColumns = columns.filter(
+        c => c.sqlType !== 'OTHER' || c.nativeType === 'COMPLEX<json>',
+      );
     }
 
     let measures = Measure.extractQueryMeasures(query);
@@ -179,6 +181,10 @@ export class QuerySource {
     return this.measures.some(m => m.name === name);
   }
 
+  public hasBaseTimeColumn(): boolean {
+    return this.baseColumns.some(column => column.isTimeColumn());
+  }
+
   public getSourceExpressionForColumn(outputName: string): SqlExpression {
     const selectExpressionsArray = this.query.getSelectExpressionsArray();
 
@@ -224,12 +230,12 @@ export class QuerySource {
     return noStarQuery.addSelect(newExpression);
   }
 
-  public addColumnAfter(neighborName: string, newExpression: SqlExpression): SqlQuery {
+  public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery {
     const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns);
     return noStarQuery.changeSelectExpressions(
       noStarQuery
         .getSelectExpressionsArray()
-        .flatMap(ex => (ex.getOutputName() === neighborName ? [ex, newExpression] : ex)),
+        .flatMap(ex => (ex.getOutputName() === neighborName ? [ex, ...newExpressions] : ex)),
     );
   }
 

web-console/src/views/explore-view/modules/grouping-table-module.tsx

@@ -105,6 +105,7 @@ ModuleRepository.registerModule<GroupingTableParameterValues>({
         count: `Show '<count> values'`,
       },
       defaultValue: 'null',
+      sticky: true,
       visible: ({ parameterValues }) => Boolean((parameterValues.showColumns || []).length),
     },
     pivotColumn: {

web-console/src/views/explore-view/modules/record-table-module.tsx

@@ -21,10 +21,9 @@ import React, { useMemo } from 'react';
 
 import { Loader } from '../../../components';
 import { useQueryManager } from '../../../hooks';
-import {
+import type { ColumnHint } from '../../../utils';
-  calculateInitPageSize,
+import { filterMap } from '../../../utils';
-  GenericOutputTable,
+import { calculateInitPageSize, GenericOutputTable } from '../components';
-} from '../components/generic-output-table/generic-output-table';
 import { ModuleRepository } from '../module-repository/module-repository';
 
 import './record-table-module.scss';
@@ -33,6 +32,7 @@ interface RecordTableParameterValues {
   maxRows: number;
   ascending: boolean;
   showTypeIcons: boolean;
+  hideNullColumns: boolean;
 }
 
 ModuleRepository.registerModule<RecordTableParameterValues>({
@@ -50,10 +50,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
     ascending: {
       type: 'boolean',
       defaultValue: false,
+      sticky: true,
     },
     showTypeIcons: {
       type: 'boolean',
       defaultValue: true,
+      sticky: true,
+    },
+    hideNullColumns: {
+      type: 'boolean',
+      label: 'Hide all null columns',
+      defaultValue: false,
+      sticky: true,
     },
   },
   component: function RecordTableModule(props) {
@@ -77,6 +85,18 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
     });
 
     const resultData = resultState.getSomeData();
+
+    let columnHints: Map<string, ColumnHint> | undefined;
+    if (parameterValues.hideNullColumns && resultData) {
+      columnHints = new Map<string, ColumnHint>(
+        filterMap(resultData.header, (column, i) =>
+          resultData.getColumnByIndex(i)?.every(v => v == null)
+            ? [column.name, { hidden: true }]
+            : undefined,
+        ),
+      );
+    }
+
     return (
       <div className="record-table-module module">
         {resultState.error ? (
@@ -84,6 +104,7 @@ ModuleRepository.registerModule<RecordTableParameterValues>({
         ) : resultData ? (
           <GenericOutputTable
             queryResult={resultData}
+            columnHints={columnHints}
             showTypeIcons={parameterValues.showTypeIcons}
             onWhereChange={setWhere}
             initPageSize={calculateInitPageSize(stage.height)}

web-console/src/views/explore-view/modules/time-chart-module.tsx

@@ -114,8 +114,9 @@ ModuleRepository.registerModule<TimeChartParameterValues>({
     },
     snappyHighlight: {
       type: 'boolean',
-      label: 'Snap highlight to nearest dates',
+      label: 'Snap highlight to granularity',
       defaultValue: true,
+      sticky: true,
     },
   },
   component: function TimeChartModule(props) {

web-console/src/views/explore-view/query-macros/aggregate.ts

@@ -20,6 +20,7 @@ import { C, SqlFunction, SqlQuery } from '@druid-toolkit/query';
 
 import { filterMap, uniq } from '../../../utils';
 import { Measure } from '../models';
+import { KNOWN_AGGREGATIONS } from '../utils';
 
 export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery {
   const usedMeasures: Map<string, boolean> = new Map();
@@ -35,7 +36,14 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery
       if (!measure) throw new Error(`${Measure.AGGREGATE} of unknown measure '${measureName}'`);
 
       usedMeasures.set(measureName, true);
-      return measure.expression;
+
+      let measureExpression = measure.expression;
+      const filter = ex.getWhereExpression();
+      if (filter) {
+        measureExpression = measureExpression.addFilterToAggregations(filter, KNOWN_AGGREGATIONS);
+      }
+
+      return measureExpression;
     }
 
     // If we encounter a (the) query with the measure definitions, and we have used those measures then expand out all the columns within them

web-console/src/views/explore-view/utils/index.ts

@@ -21,8 +21,10 @@ export * from './duration';
 export * from './filter-pattern-helpers';
 export * from './general';
 export * from './get-auto-granularity';
+export * from './known-aggregations';
 export * from './max-time-for-table';
 export * from './misc';
+export * from './query-log';
 export * from './snap-to-granularity';
 export * from './table-query';
 export * from './time-manipulation';

web-console/src/views/explore-view/utils/known-aggregations.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const KNOWN_AGGREGATIONS = [
+  'COUNT',
+  'SUM',
+  'MIN',
+  'MAX',
+  'AVG',
+  'APPROX_COUNT_DISTINCT',
+  'APPROX_COUNT_DISTINCT_BUILTIN',
+  'APPROX_QUANTILE',
+  'APPROX_QUANTILE_FIXED_BUCKETS',
+  'BLOOM_FILTER',
+  'VAR_POP',
+  'VAR_SAMP',
+  'VARIANCE',
+  'STDDEV_POP',
+  'STDDEV_SAMP',
+  'STDDEV',
+  'EARLIEST',
+  'EARLIEST_BY',
+  'LATEST',
+  'LATEST_BY',
+  'ANY_VALUE',
+  'GROUPING',
+  'ARRAY_AGG',
+  'ARRAY_AGG',
+  'ARRAY_CONCAT_AGG',
+  'ARRAY_CONCAT_AGG',
+  'STRING_AGG',
+  'LISTAGG',
+  'BIT_AND',
+  'BIT_OR',
+  'BIT_XOR',
+  'APPROX_COUNT_DISTINCT_DS_THETA',
+  'DS_THETA',
+  'APPROX_QUANTILE_DS',
+  'DS_QUANTILES_SKETCH',
+  'DS_TUPLE_DOUBLES',
+  'DS_TUPLE_DOUBLES',
+  'TDIGEST_QUANTILE',
+  'TDIGEST_GENERATE_SKETCH',
+];

web-console/src/views/explore-view/utils/query-log.ts

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+interface QueryLogEntry {
+  time: Date;
+  sqlQuery: string;
+}
+
+const MAX_QUERIES_TO_LOG = 10;
+
+export class QueryLog {
+  private readonly queryLog: QueryLogEntry[] = [];
+
+  public length(): number {
+    return this.queryLog.length;
+  }
+
+  public addQuery(sqlQuery: string): void {
+    const { queryLog } = this;
+    queryLog.unshift({ time: new Date(), sqlQuery });
+    while (queryLog.length > MAX_QUERIES_TO_LOG) queryLog.pop();
+  }
+
+  public getLastQuery(): string | undefined {
+    return this.queryLog[0]?.sqlQuery;
+  }
+
+  public getFormatted(): string {
+    return this.queryLog
+      .map(({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`)
+      .join('\n\n-----------------------------------------------------\n\n');
+  }
+}

web-console/src/views/explore-view/utils/table-query.ts

@@ -37,6 +37,7 @@ import { Measure } from '../models';
 
 import { formatDuration } from './duration';
 import { addTableScope } from './general';
+import { KNOWN_AGGREGATIONS } from './known-aggregations';
 import type { Compare } from './time-manipulation';
 import { computeWhereForCompares } from './time-manipulation';
 
@@ -48,35 +49,6 @@ export type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPerce
 
 export type RestrictTop = 'always' | 'never';
 
-const KNOWN_AGGREGATIONS = [
-  'COUNT',
-  'SUM',
-  'MIN',
-  'MAX',
-  'AVG',
-  'APPROX_COUNT_DISTINCT',
-  'APPROX_COUNT_DISTINCT_DS_HLL',
-  'APPROX_COUNT_DISTINCT_DS_THETA',
-  'DS_HLL',
-  'DS_THETA',
-  'APPROX_QUANTILE',
-  'APPROX_QUANTILE_DS',
-  'APPROX_QUANTILE_FIXED_BUCKETS',
-  'DS_QUANTILES_SKETCH',
-  'BLOOM_FILTER',
-  'TDIGEST_QUANTILE',
-  'TDIGEST_GENERATE_SKETCH',
-  'VAR_POP',
-  'VAR_SAMP',
-  'VARIANCE',
-  'STDDEV_POP',
-  'STDDEV_SAMP',
-  'STDDEV',
-  'EARLIEST',
-  'LATEST',
-  'ANY_VALUE',
-];
-
 const DRUID_DEFAULT_TOTAL_SUB_QUERY_LIMIT = 100000;
 
 const COMMON_NAME = 'common';