diff --git a/web-console/src/components/record-table-pane/record-table-pane.tsx b/web-console/src/components/record-table-pane/record-table-pane.tsx index b8629eb736c..1fe729d64d7 100644 --- a/web-console/src/components/record-table-pane/record-table-pane.tsx +++ b/web-console/src/components/record-table-pane/record-table-pane.tsx @@ -30,6 +30,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../reac import type { Pagination } from '../../utils'; import { columnToIcon, + columnToSummary, columnToWidth, filterMap, formatNumber, @@ -140,7 +141,7 @@ export const RecordTablePane = React.memo(function RecordTablePane(props: Record Header() { return (
-
+
{icon && } {h} {hasFilterOnHeader(h, i) && } diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 9388d492834..97ecc14b2fa 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -67,7 +67,7 @@ type ExecutionDestination = | { type: 'taskReport'; } - | { type: 'dataSource'; dataSource: string; exists?: boolean } + | { type: 'dataSource'; dataSource: string; loaded?: boolean } | { type: 'download' }; export type ExecutionStatus = 'RUNNING' | 'FAILED' | 'SUCCESS'; @@ -505,7 +505,7 @@ export class Execution { }); } - public markDestinationDatasourceExists(): Execution { + public markDestinationDatasourceLoaded(): Execution { const { destination } = this; if (destination?.type !== 'dataSource') return this; @@ -513,7 +513,7 @@ export class Execution { ...this.valueOf(), destination: { ...destination, - exists: true, + loaded: true, }, }); } @@ -537,7 +537,7 @@ export class Execution { const { status, destination } = this; if (status === 'SUCCESS' && destination?.type === 'dataSource') { - return Boolean(destination.exists); + return Boolean(destination.loaded); } return true; diff --git a/web-console/src/helpers/execution/general.ts b/web-console/src/helpers/execution/general.ts index bbb2eead6f0..9c6e04fd1d5 100644 --- a/web-console/src/helpers/execution/general.ts +++ b/web-console/src/helpers/execution/general.ts @@ -23,7 +23,7 @@ import type { Execution } from '../../druid-models'; import { IntermediateQueryState } from '../../utils'; import { - updateExecutionWithDatasourceExistsIfNeeded, + updateExecutionWithDatasourceLoadedIfNeeded, updateExecutionWithTaskIfNeeded, } from './sql-task-execution'; @@ -49,7 +49,7 @@ export async function executionBackgroundStatusCheck( switch (execution.engine) { case 'sql-msq-task': execution = await updateExecutionWithTaskIfNeeded(execution, cancelToken); - execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken); + execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken); break; default: diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index af5e40f315f..c8afc1bf70e 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -17,7 +17,7 @@ */ import type { AxiosResponse, CancelToken } from 'axios'; -import { L } from 'druid-query-toolkit'; +import { L, QueryResult } from 'druid-query-toolkit'; import type { QueryContext } from '../../druid-models'; import { Execution } from '../../druid-models'; @@ -31,7 +31,8 @@ import { } from '../../utils'; import { maybeGetClusterCapacity } from '../capacity'; -const WAIT_FOR_SEGMENTS_TIMEOUT = 180000; // 3 minutes to wait until segments appear +const WAIT_FOR_SEGMENT_METADATA_TIMEOUT = 180000; // 3 minutes to wait until segments appear in the metadata +const WAIT_FOR_SEGMENT_LOAD_TIMEOUT = 540000; // 9 minutes to wait for segments to load at all export interface SubmitTaskQueryOptions { query: string | Record; @@ -94,7 +95,17 @@ export async function submitTaskQuery( throw new DruidError(druidError, prefixLines); } - let execution = Execution.fromTaskSubmit(sqlTaskResp.data, sqlQuery, context); + const sqlTaskPayload = sqlTaskResp.data; + + if (!sqlTaskPayload.taskId) { + if (!Array.isArray(sqlTaskPayload)) throw new Error('unexpected task payload'); + return Execution.fromResult( + 'sql-msq-task', + QueryResult.fromRawResult(sqlTaskPayload, false, true, true, true), + ); + } + + let execution = Execution.fromTaskSubmit(sqlTaskPayload, sqlQuery, context); if (onSubmitted) { onSubmitted(execution.id); @@ -104,7 +115,7 @@ export async function submitTaskQuery( execution = execution.changeDestination({ type: 'download' }); } - execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken); + execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken); if (execution.isFullyComplete()) return execution; @@ -129,7 +140,7 @@ export async function reattachTaskExecution( try { execution = await getTaskExecution(id, undefined, cancelToken); - execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken); + execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken); } catch (e) { throw new Error(`Reattaching to query failed due to: ${e.message}`); } @@ -217,17 +228,31 @@ export async function getTaskExecution( return Execution.fromTaskStatus(statusResp.data); } -export async function updateExecutionWithDatasourceExistsIfNeeded( +export async function updateExecutionWithDatasourceLoadedIfNeeded( execution: Execution, _cancelToken?: CancelToken, ): Promise { if ( - !(execution.destination?.type === 'dataSource' && !execution.destination.exists) || + !(execution.destination?.type === 'dataSource' && !execution.destination.loaded) || execution.status !== 'SUCCESS' ) { return execution; } + const endTime = execution.getEndTime(); + if ( + !endTime || // If endTime is not set (this is not expected to happen) then just bow out + execution.stages?.getLastStage()?.partitionCount === 0 || // No data was meant to be written anyway, nothing to do + endTime.valueOf() + WAIT_FOR_SEGMENT_LOAD_TIMEOUT < Date.now() // Enough time has passed since the query ran... don't bother waiting for segments to load. + ) { + return execution.markDestinationDatasourceLoaded(); + } + + // Ideally we would have a more accurate query here, instead of + // COUNT(*) FILTER (WHERE is_published = 1 AND is_available = 0) + // we want to filter on something like + // COUNT(*) FILTER (WHERE is_should_be_available = 1 AND is_available = 0) + // `is_published` does not quite capture what we want but this is the best we have for now. const segmentCheck = await queryDruidSql({ query: `SELECT COUNT(*) AS num_segments, @@ -239,21 +264,11 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed = const numSegments: number = deepGet(segmentCheck, '0.num_segments') || 0; const loadingSegments: number = deepGet(segmentCheck, '0.loading_segments') || 0; - // There appear to be no segments either nothing was written out or they have not shown up in the metadata yet + // There appear to be no segments, since we checked above that something was written out we know that they have not shown up in the metadata yet if (numSegments === 0) { - const { stages } = execution; - if (stages) { - const lastStage = stages.getStage(stages.stageCount() - 1); - if (lastStage.partitionCount === 0) { - // No data was meant to be written anyway - return execution.markDestinationDatasourceExists(); - } - } - - const endTime = execution.getEndTime(); - if (!endTime || endTime.valueOf() + WAIT_FOR_SEGMENTS_TIMEOUT < Date.now()) { - // Enough time has passed since the query ran... give up waiting (or there is no time info). - return execution.markDestinationDatasourceExists(); + if (endTime.valueOf() + WAIT_FOR_SEGMENT_METADATA_TIMEOUT < Date.now()) { + // Enough time has passed since the query ran... give up waiting for segments to show up in metadata. + return execution.markDestinationDatasourceLoaded(); } return execution; @@ -262,7 +277,7 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed = // There are segments, and we are still waiting for some of them to load if (loadingSegments > 0) return execution; - return execution.markDestinationDatasourceExists(); + return execution.markDestinationDatasourceLoaded(); } function cancelTaskExecutionOnCancel( diff --git a/web-console/src/utils/types.ts b/web-console/src/utils/types.ts index ffb43d0115d..35324c1fb58 100644 --- a/web-console/src/utils/types.ts +++ b/web-console/src/utils/types.ts @@ -20,6 +20,13 @@ import type { IconName } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import type { Column } from 'druid-query-toolkit'; +export function columnToSummary(column: Column): string { + const lines: string[] = [column.name]; + if (column.sqlType) lines.push(`SQL type: ${column.sqlType}`); + if (column.nativeType) lines.push(`Native type: ${column.nativeType}`); + return lines.join('\n'); +} + function getEffectiveColumnType(column: Column): string | undefined { if (column.sqlType === 'TIMESTAMP') return column.sqlType; return column.nativeType || column.sqlType; @@ -42,24 +49,35 @@ export function dataTypeToIcon(dataType: string): IconName { return IconNames.FONT; case 'BIGINT': + case 'LONG': + return IconNames.NUMERICAL; + case 'DECIMAL': case 'REAL': - case 'LONG': case 'FLOAT': case 'DOUBLE': - return IconNames.NUMERICAL; + return IconNames.FLOATING_POINT; case 'ARRAY': return IconNames.ARRAY_STRING; case 'ARRAY': + return IconNames.ARRAY_NUMERIC; + case 'ARRAY': case 'ARRAY': - return IconNames.ARRAY_NUMERIC; + return IconNames.ARRAY_FLOATING_POINT; case 'COMPLEX': return IconNames.DIAGRAM_TREE; + case 'COMPLEX': + return IconNames.ALIGNMENT_HORIZONTAL_CENTER; + + case 'COMPLEX': + case 'COMPLEX': + return IconNames.IP_ADDRESS; + case 'NULL': return IconNames.CIRCLE; diff --git a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx index 810bc3bd743..c379e9322bb 100644 --- a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx +++ b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx @@ -30,7 +30,13 @@ import { BracedText, Deferred, TableCell } from '../../../../components'; import { CellFilterMenu } from '../../../../components/cell-filter-menu/cell-filter-menu'; import { ShowValueDialog } from '../../../../dialogs/show-value-dialog/show-value-dialog'; import type { QueryAction } from '../../../../utils'; -import { columnToIcon, columnToWidth, filterMap, getNumericColumnBraces } from '../../../../utils'; +import { + columnToIcon, + columnToSummary, + columnToWidth, + filterMap, + getNumericColumnBraces, +} from '../../../../utils'; import './preview-table.scss'; @@ -124,7 +130,7 @@ export const PreviewTable = React.memo(function PreviewTable(props: PreviewTable Header() { return (
onEditColumn(i)}> -
+
{icon && } {h} {hasFilterOnHeader(h, i) && ( diff --git a/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap index f09caaf1026..f1ed7216981 100644 --- a/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap +++ b/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap @@ -98,7 +98,7 @@ exports[`ColumnTree matches snapshot 1`] = ` , }, Object { - "icon": "numerical", + "icon": "floating-point", "id": "addedBy10", "label": - (stop waiting) + (skip waiting) diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx index 24a669d4a50..0e146327d83 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx @@ -57,7 +57,7 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar <> {' '} - {stages && !execution.isWaitingForQuery() ? '(stop waiting)' : '(cancel)'} + {stages && !execution.isWaitingForQuery() ? '(skip waiting)' : '(cancel)'} )} diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index 743e44174f8..8571ae59963 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -39,6 +39,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../r import type { Pagination, QueryAction } from '../../../utils'; import { columnToIcon, + columnToSummary, columnToWidth, convertToGroupByExpression, copyAndAlert, @@ -587,7 +588,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result return ( getHeaderMenu(column, i)} />}>
-
+
{icon && } {h} {hasFilterOnHeader(h, i) && (