Web console: better end of (MSQ) query segment loading UX (#14120)

* better end of query segment loading UX

* fix snapshot

* handle case when MSQ query returns results directly

* add ip address column icon

* better icons

* add variance icon

* better summary

web-console/src/components/record-table-pane/record-table-pane.tsx

@@ -30,6 +30,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../reac
 import type { Pagination } from '../../utils';
 import {
   columnToIcon,
+  columnToSummary,
   columnToWidth,
   filterMap,
   formatNumber,
@@ -140,7 +141,7 @@ export const RecordTablePane = React.memo(function RecordTablePane(props: Record
               Header() {
                 return (
                   <div className="clickable-cell">
-                    <div className="output-name">
+                    <div className="output-name" title={columnToSummary(column)}>
                       {icon && <Icon className="type-icon" icon={icon} size={12} />}
                       {h}
                       {hasFilterOnHeader(h, i) && <Icon icon={IconNames.FILTER} size={14} />}

web-console/src/druid-models/execution/execution.ts

@@ -67,7 +67,7 @@ type ExecutionDestination =
   | {
       type: 'taskReport';
     }
-  | { type: 'dataSource'; dataSource: string; exists?: boolean }
+  | { type: 'dataSource'; dataSource: string; loaded?: boolean }
   | { type: 'download' };
 
 export type ExecutionStatus = 'RUNNING' | 'FAILED' | 'SUCCESS';
@@ -505,7 +505,7 @@ export class Execution {
     });
   }
 
-  public markDestinationDatasourceExists(): Execution {
+  public markDestinationDatasourceLoaded(): Execution {
     const { destination } = this;
     if (destination?.type !== 'dataSource') return this;
 
@@ -513,7 +513,7 @@ export class Execution {
       ...this.valueOf(),
       destination: {
         ...destination,
-        exists: true,
+        loaded: true,
       },
     });
   }
@@ -537,7 +537,7 @@ export class Execution {
 
     const { status, destination } = this;
     if (status === 'SUCCESS' && destination?.type === 'dataSource') {
-      return Boolean(destination.exists);
+      return Boolean(destination.loaded);
     }
 
     return true;

web-console/src/helpers/execution/general.ts

@@ -23,7 +23,7 @@ import type { Execution } from '../../druid-models';
 import { IntermediateQueryState } from '../../utils';
 
 import {
-  updateExecutionWithDatasourceExistsIfNeeded,
+  updateExecutionWithDatasourceLoadedIfNeeded,
   updateExecutionWithTaskIfNeeded,
 } from './sql-task-execution';
 
@@ -49,7 +49,7 @@ export async function executionBackgroundStatusCheck(
   switch (execution.engine) {
     case 'sql-msq-task':
       execution = await updateExecutionWithTaskIfNeeded(execution, cancelToken);
-      execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+      execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
       break;
 
     default:

web-console/src/helpers/execution/sql-task-execution.ts

@@ -17,7 +17,7 @@
  */
 
 import type { AxiosResponse, CancelToken } from 'axios';
-import { L } from 'druid-query-toolkit';
+import { L, QueryResult } from 'druid-query-toolkit';
 
 import type { QueryContext } from '../../druid-models';
 import { Execution } from '../../druid-models';
@@ -31,7 +31,8 @@ import {
 } from '../../utils';
 import { maybeGetClusterCapacity } from '../capacity';
 
-const WAIT_FOR_SEGMENTS_TIMEOUT = 180000; // 3 minutes to wait until segments appear
+const WAIT_FOR_SEGMENT_METADATA_TIMEOUT = 180000; // 3 minutes to wait until segments appear in the metadata
+const WAIT_FOR_SEGMENT_LOAD_TIMEOUT = 540000; // 9 minutes to wait for segments to load at all
 
 export interface SubmitTaskQueryOptions {
   query: string | Record<string, any>;
@@ -94,7 +95,17 @@ export async function submitTaskQuery(
     throw new DruidError(druidError, prefixLines);
   }
 
-  let execution = Execution.fromTaskSubmit(sqlTaskResp.data, sqlQuery, context);
+  const sqlTaskPayload = sqlTaskResp.data;
+
+  if (!sqlTaskPayload.taskId) {
+    if (!Array.isArray(sqlTaskPayload)) throw new Error('unexpected task payload');
+    return Execution.fromResult(
+      'sql-msq-task',
+      QueryResult.fromRawResult(sqlTaskPayload, false, true, true, true),
+    );
+  }
+
+  let execution = Execution.fromTaskSubmit(sqlTaskPayload, sqlQuery, context);
 
   if (onSubmitted) {
     onSubmitted(execution.id);
@@ -104,7 +115,7 @@ export async function submitTaskQuery(
     execution = execution.changeDestination({ type: 'download' });
   }
 
-  execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+  execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
 
   if (execution.isFullyComplete()) return execution;
 
@@ -129,7 +140,7 @@ export async function reattachTaskExecution(
 
   try {
     execution = await getTaskExecution(id, undefined, cancelToken);
-    execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+    execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
   } catch (e) {
     throw new Error(`Reattaching to query failed due to: ${e.message}`);
   }
@@ -217,17 +228,31 @@ export async function getTaskExecution(
   return Execution.fromTaskStatus(statusResp.data);
 }
 
-export async function updateExecutionWithDatasourceExistsIfNeeded(
+export async function updateExecutionWithDatasourceLoadedIfNeeded(
   execution: Execution,
   _cancelToken?: CancelToken,
 ): Promise<Execution> {
   if (
-    !(execution.destination?.type === 'dataSource' && !execution.destination.exists) ||
+    !(execution.destination?.type === 'dataSource' && !execution.destination.loaded) ||
     execution.status !== 'SUCCESS'
   ) {
     return execution;
   }
 
+  const endTime = execution.getEndTime();
+  if (
+    !endTime || // If endTime is not set (this is not expected to happen) then just bow out
+    execution.stages?.getLastStage()?.partitionCount === 0 || // No data was meant to be written anyway, nothing to do
+    endTime.valueOf() + WAIT_FOR_SEGMENT_LOAD_TIMEOUT < Date.now() // Enough time has passed since the query ran... don't bother waiting for segments to load.
+  ) {
+    return execution.markDestinationDatasourceLoaded();
+  }
+
+  // Ideally we would have a more accurate query here, instead of
+  //   COUNT(*) FILTER (WHERE is_published = 1 AND is_available = 0)
+  // we want to filter on something like
+  //   COUNT(*) FILTER (WHERE is_should_be_available = 1 AND is_available = 0)
+  // `is_published` does not quite capture what we want but this is the best we have for now.
   const segmentCheck = await queryDruidSql({
     query: `SELECT
   COUNT(*) AS num_segments,
@@ -239,21 +264,11 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed =
   const numSegments: number = deepGet(segmentCheck, '0.num_segments') || 0;
   const loadingSegments: number = deepGet(segmentCheck, '0.loading_segments') || 0;
 
-  // There appear to be no segments either nothing was written out or they have not shown up in the metadata yet
+  // There appear to be no segments, since we checked above that something was written out we know that they have not shown up in the metadata yet
   if (numSegments === 0) {
-    const { stages } = execution;
-    if (stages) {
-      const lastStage = stages.getStage(stages.stageCount() - 1);
-      if (lastStage.partitionCount === 0) {
-        // No data was meant to be written anyway
-        return execution.markDestinationDatasourceExists();
-      }
-    }
-
-    const endTime = execution.getEndTime();
-    if (!endTime || endTime.valueOf() + WAIT_FOR_SEGMENTS_TIMEOUT < Date.now()) {
-      // Enough time has passed since the query ran... give up waiting (or there is no time info).
-      return execution.markDestinationDatasourceExists();
+    if (endTime.valueOf() + WAIT_FOR_SEGMENT_METADATA_TIMEOUT < Date.now()) {
+      // Enough time has passed since the query ran... give up waiting for segments to show up in metadata.
+      return execution.markDestinationDatasourceLoaded();
     }
 
     return execution;
@@ -262,7 +277,7 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed =
   // There are segments, and we are still waiting for some of them to load
   if (loadingSegments > 0) return execution;
 
-  return execution.markDestinationDatasourceExists();
+  return execution.markDestinationDatasourceLoaded();
 }
 
 function cancelTaskExecutionOnCancel(

web-console/src/utils/types.ts

@@ -20,6 +20,13 @@ import type { IconName } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import type { Column } from 'druid-query-toolkit';
 
+export function columnToSummary(column: Column): string {
+  const lines: string[] = [column.name];
+  if (column.sqlType) lines.push(`SQL type: ${column.sqlType}`);
+  if (column.nativeType) lines.push(`Native type: ${column.nativeType}`);
+  return lines.join('\n');
+}
+
 function getEffectiveColumnType(column: Column): string | undefined {
   if (column.sqlType === 'TIMESTAMP') return column.sqlType;
   return column.nativeType || column.sqlType;
@@ -42,24 +49,35 @@ export function dataTypeToIcon(dataType: string): IconName {
       return IconNames.FONT;
 
     case 'BIGINT':
+    case 'LONG':
+      return IconNames.NUMERICAL;
+
     case 'DECIMAL':
     case 'REAL':
-    case 'LONG':
     case 'FLOAT':
     case 'DOUBLE':
-      return IconNames.NUMERICAL;
+      return IconNames.FLOATING_POINT;
 
     case 'ARRAY<STRING>':
       return IconNames.ARRAY_STRING;
 
     case 'ARRAY<LONG>':
+      return IconNames.ARRAY_NUMERIC;
+
     case 'ARRAY<FLOAT>':
     case 'ARRAY<DOUBLE>':
-      return IconNames.ARRAY_NUMERIC;
+      return IconNames.ARRAY_FLOATING_POINT;
 
     case 'COMPLEX<JSON>':
       return IconNames.DIAGRAM_TREE;
 
+    case 'COMPLEX<VARIANCE>':
+      return IconNames.ALIGNMENT_HORIZONTAL_CENTER;
+
+    case 'COMPLEX<IPADDRESS>':
+    case 'COMPLEX<IPPREFIX>':
+      return IconNames.IP_ADDRESS;
+
     case 'NULL':
       return IconNames.CIRCLE;
 

web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx

@@ -30,7 +30,13 @@ import { BracedText, Deferred, TableCell } from '../../../../components';
 import { CellFilterMenu } from '../../../../components/cell-filter-menu/cell-filter-menu';
 import { ShowValueDialog } from '../../../../dialogs/show-value-dialog/show-value-dialog';
 import type { QueryAction } from '../../../../utils';
-import { columnToIcon, columnToWidth, filterMap, getNumericColumnBraces } from '../../../../utils';
+import {
+  columnToIcon,
+  columnToSummary,
+  columnToWidth,
+  filterMap,
+  getNumericColumnBraces,
+} from '../../../../utils';
 
 import './preview-table.scss';
 
@@ -124,7 +130,7 @@ export const PreviewTable = React.memo(function PreviewTable(props: PreviewTable
             Header() {
               return (
                 <div className="header-wrapper" onClick={() => onEditColumn(i)}>
-                  <div className="output-name">
+                  <div className="output-name" title={columnToSummary(column)}>
                     {icon && <Icon className="type-icon" icon={icon} size={12} />}
                     {h}
                     {hasFilterOnHeader(h, i) && (

web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap

@@ -98,7 +98,7 @@ exports[`ColumnTree matches snapshot 1`] = `
                 </Blueprint4.Popover2>,
               },
               Object {
-                "icon": "numerical",
+                "icon": "floating-point",
                 "id": "addedBy10",
                 "label": <Blueprint4.Popover2
                   autoFocus={false}

web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap

@@ -12,7 +12,7 @@ exports[`ExecutionProgressBarPane matches snapshot 1`] = `
         className="cancel"
         onClick={[Function]}
       >
-        (stop waiting)
+        (skip waiting)
       </span>
     </React.Fragment>
   </Unknown>

web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx

@@ -57,7 +57,7 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar
           <>
             {' '}
             <span className="cancel" onClick={cancelMaybeConfirm}>
-              {stages && !execution.isWaitingForQuery() ? '(stop waiting)' : '(cancel)'}
+              {stages && !execution.isWaitingForQuery() ? '(skip waiting)' : '(cancel)'}
             </span>
           </>
         )}

web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx

@@ -39,6 +39,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../r
 import type { Pagination, QueryAction } from '../../../utils';
 import {
   columnToIcon,
+  columnToSummary,
   columnToWidth,
   convertToGroupByExpression,
   copyAndAlert,
@@ -587,7 +588,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result
                 return (
                   <Popover2 content={<Deferred content={() => getHeaderMenu(column, i)} />}>
                     <div className="clickable-cell">
-                      <div className="output-name">
+                      <div className="output-name" title={columnToSummary(column)}>
                         {icon && <Icon className="type-icon" icon={icon} size={12} />}
                         {h}
                         {hasFilterOnHeader(h, i) && (