From 8c170f7d0e2763cf11282ea17a602397c5efe151 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 1 Aug 2024 02:30:30 -0700 Subject: [PATCH] Web console: use stages, counters, and warnings from the new detailed status API (#16809) * stages and counters can be seen on the status reponse * warnings are exposed also * mark as msq when attached * update snapshots * download CSV/TSV null as empty cell --- .../async-query/async-query.mock.ts | 674 +++++++++++++- .../druid-models/async-query/async-query.ts | 5 + .../druid-models/execution/execution.spec.ts | 836 +++++++++++++++++- .../src/druid-models/execution/execution.ts | 6 +- web-console/src/druid-models/stages/stages.ts | 13 +- web-console/src/druid-models/task/task.ts | 4 +- .../workbench-query/workbench-query.ts | 3 +- .../helpers/execution/sql-task-execution.ts | 2 +- web-console/src/utils/download.spec.ts | 5 +- web-console/src/utils/download.ts | 2 + .../recent-query-task-panel.tsx | 2 +- .../views/workbench-view/workbench-view.tsx | 5 +- 12 files changed, 1515 insertions(+), 42 deletions(-) diff --git a/web-console/src/druid-models/async-query/async-query.mock.ts b/web-console/src/druid-models/async-query/async-query.mock.ts index 906f82a753f..c3c0e1d1926 100644 --- a/web-console/src/druid-models/async-query/async-query.mock.ts +++ b/web-console/src/druid-models/async-query/async-query.mock.ts @@ -29,9 +29,9 @@ LIMIT 2 */ export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = { - queryId: 'query-ad84d20a-c331-4ee9-ac59-83024e369cf1', + queryId: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349', state: 'SUCCESS', - createdAt: '2023-07-05T21:33:19.147Z', + createdAt: '2024-07-27T02:39:22.230Z', schema: [ { name: 'channel', @@ -44,10 +44,10 @@ export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = { nativeType: 'LONG', }, ], - durationMs: 29168, + durationMs: 7183, result: { numTotalRows: 2, - totalSizeInBytes: 116, + totalSizeInBytes: 150, dataSource: '__query_select', sampleRecords: [ ['#en.wikipedia', 6650], @@ -55,12 +55,395 @@ export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = { ], pages: [ { - numRows: 2, - sizeInBytes: 116, id: 0, + numRows: 2, + sizeInBytes: 150, }, ], }, + stages: [ + { + stageNumber: 0, + definition: { + id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_0', + input: [ + { + type: 'table', + dataSource: 'wikipedia', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + ], + processor: { + type: 'groupByPreShuffle', + query: { + queryType: 'groupBy', + dataSource: { + type: 'inputNumber', + inputNumber: 0, + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + granularity: { + type: 'all', + }, + dimensions: [ + { + type: 'default', + dimension: 'channel', + outputName: 'd0', + outputType: 'STRING', + }, + ], + aggregations: [ + { + type: 'count', + name: 'a0', + }, + ], + limitSpec: { + type: 'default', + columns: [ + { + dimension: 'a0', + direction: 'descending', + dimensionOrder: { + type: 'numeric', + }, + }, + ], + limit: 2, + }, + context: { + __resultFormat: 'array', + __user: 'allowAll', + executionMode: 'async', + finalize: true, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349', + sqlOuterLimit: 1001, + sqlQueryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349', + sqlStringifyArrays: false, + }, + }, + }, + signature: [ + { + name: 'd0', + type: 'STRING', + }, + { + name: 'a0', + type: 'LONG', + }, + ], + shuffleSpec: { + type: 'maxCount', + clusterBy: { + columns: [ + { + columnName: 'd0', + order: 'ASCENDING', + }, + ], + }, + partitions: 1, + aggregate: true, + }, + maxWorkerCount: 1, + }, + phase: 'FINISHED', + workerCount: 1, + partitionCount: 1, + shuffle: 'globalSort', + output: 'localStorage', + startTime: '2024-07-27T02:39:24.713Z', + duration: 3384, + sort: true, + }, + { + stageNumber: 1, + definition: { + id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_1', + input: [ + { + type: 'stage', + stage: 0, + }, + ], + processor: { + type: 'groupByPostShuffle', + query: { + queryType: 'groupBy', + dataSource: { + type: 'inputNumber', + inputNumber: 0, + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + granularity: { + type: 'all', + }, + dimensions: [ + { + type: 'default', + dimension: 'channel', + outputName: 'd0', + outputType: 'STRING', + }, + ], + aggregations: [ + { + type: 'count', + name: 'a0', + }, + ], + limitSpec: { + type: 'default', + columns: [ + { + dimension: 'a0', + direction: 'descending', + dimensionOrder: { + type: 'numeric', + }, + }, + ], + limit: 2, + }, + context: { + __resultFormat: 'array', + __user: 'allowAll', + executionMode: 'async', + finalize: true, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349', + sqlOuterLimit: 1001, + sqlQueryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349', + sqlStringifyArrays: false, + }, + }, + }, + signature: [ + { + name: 'a0', + type: 'LONG', + }, + { + name: '__boost', + type: 'LONG', + }, + { + name: 'd0', + type: 'STRING', + }, + ], + shuffleSpec: { + type: 'maxCount', + clusterBy: { + columns: [ + { + columnName: 'a0', + order: 'DESCENDING', + }, + { + columnName: '__boost', + order: 'ASCENDING', + }, + ], + }, + partitions: 1, + }, + maxWorkerCount: 1, + }, + phase: 'FINISHED', + workerCount: 1, + partitionCount: 1, + shuffle: 'globalSort', + output: 'localStorage', + startTime: '2024-07-27T02:39:28.089Z', + duration: 26, + sort: true, + }, + { + stageNumber: 2, + definition: { + id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_2', + input: [ + { + type: 'stage', + stage: 1, + }, + ], + processor: { + type: 'limit', + limit: 2, + }, + signature: [ + { + name: 'a0', + type: 'LONG', + }, + { + name: '__boost', + type: 'LONG', + }, + { + name: 'd0', + type: 'STRING', + }, + ], + shuffleSpec: { + type: 'maxCount', + clusterBy: { + columns: [ + { + columnName: 'a0', + order: 'DESCENDING', + }, + { + columnName: '__boost', + order: 'ASCENDING', + }, + ], + }, + partitions: 1, + }, + maxWorkerCount: 1, + }, + phase: 'FINISHED', + workerCount: 1, + partitionCount: 1, + shuffle: 'globalSort', + output: 'localStorage', + startTime: '2024-07-27T02:39:28.112Z', + duration: 12, + sort: true, + }, + ], + counters: { + '0': { + '0': { + input0: { + type: 'channel', + rows: [24433], + bytes: [6525055], + files: [1], + totalFiles: [1], + }, + output: { + type: 'channel', + rows: [51], + bytes: [2335], + frames: [1], + }, + shuffle: { + type: 'channel', + rows: [51], + bytes: [2131], + frames: [1], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + totalMergersForUltimateLevel: 1, + progressDigest: 1.0, + }, + }, + }, + '1': { + '0': { + input0: { + type: 'channel', + rows: [51], + bytes: [2131], + frames: [1], + }, + output: { + type: 'channel', + rows: [51], + bytes: [2998], + frames: [1], + }, + shuffle: { + type: 'channel', + rows: [51], + bytes: [2794], + frames: [1], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + totalMergersForUltimateLevel: 1, + progressDigest: 1.0, + }, + }, + }, + '2': { + '0': { + input0: { + type: 'channel', + rows: [51], + bytes: [2794], + frames: [1], + }, + output: { + type: 'channel', + rows: [2], + bytes: [150], + frames: [1], + }, + shuffle: { + type: 'channel', + rows: [2], + bytes: [142], + frames: [1], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + totalMergersForUltimateLevel: 1, + progressDigest: 1.0, + }, + }, + }, + }, + warnings: [], }; /* @@ -82,18 +465,285 @@ PARTITIONED BY DAY */ export const FAILED_ASYNC_STATUS: AsyncStatusResponse = { - queryId: 'query-36ea273a-bd6d-48de-b890-2d853d879bf8', + queryId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4', state: 'FAILED', - createdAt: '2023-07-05T21:40:39.986Z', - durationMs: 11217, + createdAt: '2024-07-26T18:04:59.873Z', + durationMs: 6954, errorDetails: { error: 'druidException', - errorCode: 'UnknownError', + errorCode: 'TooManyWarnings', persona: 'USER', category: 'UNCATEGORIZED', - errorMessage: 'java.io.UncheckedIOException: /', + errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)', context: { - message: 'java.io.UncheckedIOException: /', + maxWarnings: '2', + rootErrorCode: 'CannotParseExternalData', }, }, + stages: [ + { + stageNumber: 0, + definition: { + id: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4_0', + input: [ + { + type: 'external', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'], + }, + inputFormat: { + type: 'json', + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', + }, + ], + }, + ], + processor: { + type: 'scan', + query: { + queryType: 'scan', + dataSource: { + type: 'external', + inputSource: { + type: 'http', + uris: [ + 'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json', + ], + }, + inputFormat: { + type: 'json', + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', + }, + ], + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + virtualColumns: [ + { + type: 'expression', + name: 'v0', + expression: 'timestamp_parse("timestamp",null,\'UTC\')', + outputType: 'LONG', + }, + ], + resultFormat: 'compactedList', + columns: ['agent_type', 'v0'], + context: { + __resultFormat: 'array', + __timeColumn: 'v0', + __user: 'allowAll', + executionMode: 'async', + finalize: false, + finalizeAggregations: false, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 2, + queryId: 'ea3e36df-ad67-4870-b136-f5616b17d9c4', + scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', + sqlInsertSegmentGranularity: '"DAY"', + sqlQueryId: 'ea3e36df-ad67-4870-b136-f5616b17d9c4', + sqlReplaceTimeChunks: 'all', + sqlStringifyArrays: false, + waitUntilSegmentsLoad: true, + }, + columnTypes: ['STRING', 'LONG'], + granularity: { + type: 'all', + }, + legacy: false, + }, + }, + signature: [ + { + name: '__bucket', + type: 'LONG', + }, + { + name: '__boost', + type: 'LONG', + }, + { + name: 'agent_type', + type: 'STRING', + }, + { + name: 'v0', + type: 'LONG', + }, + ], + shuffleSpec: { + type: 'targetSize', + clusterBy: { + columns: [ + { + columnName: '__bucket', + order: 'ASCENDING', + }, + { + columnName: '__boost', + order: 'ASCENDING', + }, + ], + bucketByCount: 1, + }, + targetSize: 3000000, + }, + maxWorkerCount: 1, + shuffleCheckHasMultipleValues: true, + }, + phase: 'FAILED', + workerCount: 1, + shuffle: 'globalSort', + output: 'localStorage', + startTime: '2024-07-26T18:05:02.399Z', + duration: 4056, + sort: true, + }, + { + stageNumber: 1, + definition: { + id: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4_1', + input: [ + { + type: 'stage', + stage: 0, + }, + ], + processor: { + type: 'segmentGenerator', + dataSchema: { + dataSource: 'kttm-blank-lines', + timestampSpec: { + column: '__time', + format: 'millis', + missingValue: null, + }, + dimensionsSpec: { + dimensions: [ + { + type: 'string', + name: 'agent_type', + multiValueHandling: 'SORTED_ARRAY', + createBitmapIndex: true, + }, + ], + dimensionExclusions: ['__time'], + includeAllDimensions: false, + useSchemaDiscovery: false, + }, + metricsSpec: [], + granularitySpec: { + type: 'arbitrary', + queryGranularity: { + type: 'none', + }, + rollup: false, + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + transformSpec: { + filter: null, + transforms: [], + }, + }, + columnMappings: [ + { + queryColumn: 'v0', + outputColumn: '__time', + }, + { + queryColumn: 'agent_type', + outputColumn: 'agent_type', + }, + ], + tuningConfig: { + maxNumWorkers: 1, + maxRowsInMemory: 100000, + rowsPerSegment: 3000000, + }, + }, + signature: [], + maxWorkerCount: 1, + }, + }, + ], + counters: { + '0': { + '0': { + input0: { + type: 'channel', + rows: [10], + bytes: [7658], + files: [1], + totalFiles: [1], + }, + output: { + type: 'channel', + rows: [10], + bytes: [712], + frames: [1], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': -1, + }, + levelToMergedBatches: {}, + totalMergersForUltimateLevel: -1, + progressDigest: 0.0, + }, + warnings: { + type: 'warnings', + CannotParseExternalData: 3, + }, + }, + }, + }, + warnings: [ + { + taskId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0', + host: 'localhost:8101', + stageNumber: 0, + error: { + errorCode: 'CannotParseExternalData', + errorMessage: + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)', + }, + exceptionStackTrace: + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (byte[])""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n', + }, + { + taskId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0', + host: 'localhost:8101', + stageNumber: 0, + error: { + errorCode: 'CannotParseExternalData', + errorMessage: + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)', + }, + exceptionStackTrace: + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (byte[])""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n', + }, + ], }; diff --git a/web-console/src/druid-models/async-query/async-query.ts b/web-console/src/druid-models/async-query/async-query.ts index e823ba6a17e..71eec372515 100644 --- a/web-console/src/druid-models/async-query/async-query.ts +++ b/web-console/src/druid-models/async-query/async-query.ts @@ -17,6 +17,8 @@ */ import type { ErrorResponse } from '../../utils'; +import type { Counters, StageDefinition } from '../stages/stages'; +import type { MsqTaskErrorReport } from '../task/task'; export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'SUCCESS' | 'FAILED'; @@ -37,5 +39,8 @@ export interface AsyncStatusResponse { sizeInBytes: number; }[]; }; + stages?: StageDefinition[]; + counters?: Counters; errorDetails?: ErrorResponse; + warnings?: MsqTaskErrorReport[]; } diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index 275de4f1757..2938351a22a 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -622,13 +622,13 @@ describe('Execution', () => { { "id": 0, "numRows": 2, - "sizeInBytes": 116, + "sizeInBytes": 150, }, ], - "duration": 29168, + "duration": 7183, "engine": "sql-msq-task", "error": undefined, - "id": "query-ad84d20a-c331-4ee9-ac59-83024e369cf1", + "id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349", "nativeQuery": undefined, "queryContext": undefined, "result": _QueryResult { @@ -663,8 +663,453 @@ describe('Execution', () => { }, "segmentStatus": undefined, "sqlQuery": undefined, - "stages": undefined, - "startTime": 2023-07-05T21:33:19.147Z, + "stages": Stages { + "counters": { + "0": { + "0": { + "input0": { + "bytes": [ + 6525055, + ], + "files": [ + 1, + ], + "rows": [ + 24433, + ], + "totalFiles": [ + 1, + ], + "type": "channel", + }, + "output": { + "bytes": [ + 2335, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "shuffle": { + "bytes": [ + 2131, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "sortProgress": { + "levelToMergedBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "levelToTotalBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "progressDigest": 1, + "totalMergersForUltimateLevel": 1, + "totalMergingLevels": 3, + "type": "sortProgress", + }, + }, + }, + "1": { + "0": { + "input0": { + "bytes": [ + 2131, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "output": { + "bytes": [ + 2998, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "shuffle": { + "bytes": [ + 2794, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "sortProgress": { + "levelToMergedBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "levelToTotalBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "progressDigest": 1, + "totalMergersForUltimateLevel": 1, + "totalMergingLevels": 3, + "type": "sortProgress", + }, + }, + }, + "2": { + "0": { + "input0": { + "bytes": [ + 2794, + ], + "frames": [ + 1, + ], + "rows": [ + 51, + ], + "type": "channel", + }, + "output": { + "bytes": [ + 150, + ], + "frames": [ + 1, + ], + "rows": [ + 2, + ], + "type": "channel", + }, + "shuffle": { + "bytes": [ + 142, + ], + "frames": [ + 1, + ], + "rows": [ + 2, + ], + "type": "channel", + }, + "sortProgress": { + "levelToMergedBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "levelToTotalBatches": { + "0": 1, + "1": 1, + "2": 1, + }, + "progressDigest": 1, + "totalMergersForUltimateLevel": 1, + "totalMergingLevels": 3, + "type": "sortProgress", + }, + }, + }, + }, + "stages": [ + { + "definition": { + "id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_0", + "input": [ + { + "dataSource": "wikipedia", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z", + ], + "type": "table", + }, + ], + "maxWorkerCount": 1, + "processor": { + "query": { + "aggregations": [ + { + "name": "a0", + "type": "count", + }, + ], + "context": { + "__resultFormat": "array", + "__user": "allowAll", + "executionMode": "async", + "finalize": true, + "maxNumTasks": 2, + "maxParseExceptions": 0, + "queryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349", + "sqlOuterLimit": 1001, + "sqlQueryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349", + "sqlStringifyArrays": false, + }, + "dataSource": { + "inputNumber": 0, + "type": "inputNumber", + }, + "dimensions": [ + { + "dimension": "channel", + "outputName": "d0", + "outputType": "STRING", + "type": "default", + }, + ], + "granularity": { + "type": "all", + }, + "intervals": { + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z", + ], + "type": "intervals", + }, + "limitSpec": { + "columns": [ + { + "dimension": "a0", + "dimensionOrder": { + "type": "numeric", + }, + "direction": "descending", + }, + ], + "limit": 2, + "type": "default", + }, + "queryType": "groupBy", + }, + "type": "groupByPreShuffle", + }, + "shuffleSpec": { + "aggregate": true, + "clusterBy": { + "columns": [ + { + "columnName": "d0", + "order": "ASCENDING", + }, + ], + }, + "partitions": 1, + "type": "maxCount", + }, + "signature": [ + { + "name": "d0", + "type": "STRING", + }, + { + "name": "a0", + "type": "LONG", + }, + ], + }, + "duration": 3384, + "output": "localStorage", + "partitionCount": 1, + "phase": "FINISHED", + "shuffle": "globalSort", + "sort": true, + "stageNumber": 0, + "startTime": "2024-07-27T02:39:24.713Z", + "workerCount": 1, + }, + { + "definition": { + "id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_1", + "input": [ + { + "stage": 0, + "type": "stage", + }, + ], + "maxWorkerCount": 1, + "processor": { + "query": { + "aggregations": [ + { + "name": "a0", + "type": "count", + }, + ], + "context": { + "__resultFormat": "array", + "__user": "allowAll", + "executionMode": "async", + "finalize": true, + "maxNumTasks": 2, + "maxParseExceptions": 0, + "queryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349", + "sqlOuterLimit": 1001, + "sqlQueryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349", + "sqlStringifyArrays": false, + }, + "dataSource": { + "inputNumber": 0, + "type": "inputNumber", + }, + "dimensions": [ + { + "dimension": "channel", + "outputName": "d0", + "outputType": "STRING", + "type": "default", + }, + ], + "granularity": { + "type": "all", + }, + "intervals": { + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z", + ], + "type": "intervals", + }, + "limitSpec": { + "columns": [ + { + "dimension": "a0", + "dimensionOrder": { + "type": "numeric", + }, + "direction": "descending", + }, + ], + "limit": 2, + "type": "default", + }, + "queryType": "groupBy", + }, + "type": "groupByPostShuffle", + }, + "shuffleSpec": { + "clusterBy": { + "columns": [ + { + "columnName": "a0", + "order": "DESCENDING", + }, + { + "columnName": "__boost", + "order": "ASCENDING", + }, + ], + }, + "partitions": 1, + "type": "maxCount", + }, + "signature": [ + { + "name": "a0", + "type": "LONG", + }, + { + "name": "__boost", + "type": "LONG", + }, + { + "name": "d0", + "type": "STRING", + }, + ], + }, + "duration": 26, + "output": "localStorage", + "partitionCount": 1, + "phase": "FINISHED", + "shuffle": "globalSort", + "sort": true, + "stageNumber": 1, + "startTime": "2024-07-27T02:39:28.089Z", + "workerCount": 1, + }, + { + "definition": { + "id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_2", + "input": [ + { + "stage": 1, + "type": "stage", + }, + ], + "maxWorkerCount": 1, + "processor": { + "limit": 2, + "type": "limit", + }, + "shuffleSpec": { + "clusterBy": { + "columns": [ + { + "columnName": "a0", + "order": "DESCENDING", + }, + { + "columnName": "__boost", + "order": "ASCENDING", + }, + ], + }, + "partitions": 1, + "type": "maxCount", + }, + "signature": [ + { + "name": "a0", + "type": "LONG", + }, + { + "name": "__boost", + "type": "LONG", + }, + { + "name": "d0", + "type": "STRING", + }, + ], + }, + "duration": 12, + "output": "localStorage", + "partitionCount": 1, + "phase": "FINISHED", + "shuffle": "globalSort", + "sort": true, + "stageNumber": 2, + "startTime": "2024-07-27T02:39:28.112Z", + "workerCount": 1, + }, + ], + }, + "startTime": 2024-07-27T02:39:22.230Z, "status": "SUCCESS", "usageInfo": undefined, "warnings": undefined, @@ -679,32 +1124,395 @@ describe('Execution', () => { "capacityInfo": undefined, "destination": undefined, "destinationPages": undefined, - "duration": 11217, + "duration": 6954, "engine": "sql-msq-task", "error": { "error": { "category": "UNCATEGORIZED", "context": { - "message": "java.io.UncheckedIOException: /", + "maxWarnings": "2", + "rootErrorCode": "CannotParseExternalData", }, "error": "druidException", - "errorCode": "UnknownError", - "errorMessage": "java.io.UncheckedIOException: /", + "errorCode": "TooManyWarnings", + "errorMessage": "Too many warnings of type CannotParseExternalData generated (max = 2)", "persona": "USER", }, - "taskId": "query-36ea273a-bd6d-48de-b890-2d853d879bf8", + "taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4", }, - "id": "query-36ea273a-bd6d-48de-b890-2d853d879bf8", + "id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4", "nativeQuery": undefined, "queryContext": undefined, "result": undefined, "segmentStatus": undefined, "sqlQuery": undefined, - "stages": undefined, - "startTime": 2023-07-05T21:40:39.986Z, + "stages": Stages { + "counters": { + "0": { + "0": { + "input0": { + "bytes": [ + 7658, + ], + "files": [ + 1, + ], + "rows": [ + 10, + ], + "totalFiles": [ + 1, + ], + "type": "channel", + }, + "output": { + "bytes": [ + 712, + ], + "frames": [ + 1, + ], + "rows": [ + 10, + ], + "type": "channel", + }, + "sortProgress": { + "levelToMergedBatches": {}, + "levelToTotalBatches": { + "0": 1, + "1": 1, + "2": -1, + }, + "progressDigest": 0, + "totalMergersForUltimateLevel": -1, + "totalMergingLevels": 3, + "type": "sortProgress", + }, + "warnings": { + "CannotParseExternalData": 3, + "type": "warnings", + }, + }, + }, + }, + "stages": [ + { + "definition": { + "id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4_0", + "input": [ + { + "inputFormat": { + "type": "json", + }, + "inputSource": { + "type": "http", + "uris": [ + "https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json", + ], + }, + "signature": [ + { + "name": "timestamp", + "type": "STRING", + }, + { + "name": "agent_type", + "type": "STRING", + }, + ], + "type": "external", + }, + ], + "maxWorkerCount": 1, + "processor": { + "query": { + "columnTypes": [ + "STRING", + "LONG", + ], + "columns": [ + "agent_type", + "v0", + ], + "context": { + "__resultFormat": "array", + "__timeColumn": "v0", + "__user": "allowAll", + "executionMode": "async", + "finalize": false, + "finalizeAggregations": false, + "groupByEnableMultiValueUnnesting": false, + "maxNumTasks": 2, + "maxParseExceptions": 2, + "queryId": "ea3e36df-ad67-4870-b136-f5616b17d9c4", + "scanSignature": "[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]", + "sqlInsertSegmentGranularity": ""DAY"", + "sqlQueryId": "ea3e36df-ad67-4870-b136-f5616b17d9c4", + "sqlReplaceTimeChunks": "all", + "sqlStringifyArrays": false, + "waitUntilSegmentsLoad": true, + }, + "dataSource": { + "inputFormat": { + "type": "json", + }, + "inputSource": { + "type": "http", + "uris": [ + "https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json", + ], + }, + "signature": [ + { + "name": "timestamp", + "type": "STRING", + }, + { + "name": "agent_type", + "type": "STRING", + }, + ], + "type": "external", + }, + "granularity": { + "type": "all", + }, + "intervals": { + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z", + ], + "type": "intervals", + }, + "legacy": false, + "queryType": "scan", + "resultFormat": "compactedList", + "virtualColumns": [ + { + "expression": "timestamp_parse("timestamp",null,'UTC')", + "name": "v0", + "outputType": "LONG", + "type": "expression", + }, + ], + }, + "type": "scan", + }, + "shuffleCheckHasMultipleValues": true, + "shuffleSpec": { + "clusterBy": { + "bucketByCount": 1, + "columns": [ + { + "columnName": "__bucket", + "order": "ASCENDING", + }, + { + "columnName": "__boost", + "order": "ASCENDING", + }, + ], + }, + "targetSize": 3000000, + "type": "targetSize", + }, + "signature": [ + { + "name": "__bucket", + "type": "LONG", + }, + { + "name": "__boost", + "type": "LONG", + }, + { + "name": "agent_type", + "type": "STRING", + }, + { + "name": "v0", + "type": "LONG", + }, + ], + }, + "duration": 4056, + "output": "localStorage", + "phase": "FAILED", + "shuffle": "globalSort", + "sort": true, + "stageNumber": 0, + "startTime": "2024-07-26T18:05:02.399Z", + "workerCount": 1, + }, + { + "definition": { + "id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4_1", + "input": [ + { + "stage": 0, + "type": "stage", + }, + ], + "maxWorkerCount": 1, + "processor": { + "columnMappings": [ + { + "outputColumn": "__time", + "queryColumn": "v0", + }, + { + "outputColumn": "agent_type", + "queryColumn": "agent_type", + }, + ], + "dataSchema": { + "dataSource": "kttm-blank-lines", + "dimensionsSpec": { + "dimensionExclusions": [ + "__time", + ], + "dimensions": [ + { + "createBitmapIndex": true, + "multiValueHandling": "SORTED_ARRAY", + "name": "agent_type", + "type": "string", + }, + ], + "includeAllDimensions": false, + "useSchemaDiscovery": false, + }, + "granularitySpec": { + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z", + ], + "queryGranularity": { + "type": "none", + }, + "rollup": false, + "type": "arbitrary", + }, + "metricsSpec": [], + "timestampSpec": { + "column": "__time", + "format": "millis", + "missingValue": null, + }, + "transformSpec": { + "filter": null, + "transforms": [], + }, + }, + "tuningConfig": { + "maxNumWorkers": 1, + "maxRowsInMemory": 100000, + "rowsPerSegment": 3000000, + }, + "type": "segmentGenerator", + }, + "signature": [], + }, + "stageNumber": 1, + }, + ], + }, + "startTime": 2024-07-26T18:04:59.873Z, "status": "FAILED", "usageInfo": undefined, - "warnings": undefined, + "warnings": [ + { + "error": { + "errorCode": "CannotParseExternalData", + "errorMessage": "Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)", + }, + "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3) + at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94) + at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) + at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) + at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157) + at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) + Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input + at [Source: (byte[])""; line: 1, column: 0] + at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609) + at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) + at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) + at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) + ... 24 more + ", + "host": "localhost:8101", + "stageNumber": 0, + "taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0", + }, + { + "error": { + "errorCode": "CannotParseExternalData", + "errorMessage": "Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)", + }, + "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7) + at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94) + at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) + at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) + at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157) + at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) + Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input + at [Source: (byte[])""; line: 1, column: 0] + at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609) + at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) + at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) + at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) + ... 24 more + ", + "host": "localhost:8101", + "stageNumber": 0, + "taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0", + }, + ], } `); }); diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index dcb5b2d1b93..45bfc517538 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -194,7 +194,7 @@ export interface ExecutionValue { export class Execution { static USE_TASK_PAYLOAD = true; - static USE_TASK_REPORTS = true; + static USE_TASK_REPORTS = false; static INLINE_DATASOURCE_MARKER = '__query_select'; static getClusterCapacity: (() => Promise) | undefined = @@ -235,7 +235,7 @@ export class Execution { sqlQuery?: string, queryContext?: QueryContext, ): Execution { - const { queryId, schema, result, errorDetails } = asyncSubmitResult; + const { queryId, schema, result, errorDetails, stages, counters, warnings } = asyncSubmitResult; let queryResult: QueryResult | undefined; if (schema && result?.sampleRecords) { @@ -263,6 +263,8 @@ export class Execution { status: Execution.normalizeAsyncState(asyncSubmitResult.state), sqlQuery, queryContext, + stages: Array.isArray(stages) && counters ? new Stages(stages, counters) : undefined, + warnings: Array.isArray(warnings) ? warnings : undefined, error: executionError, destination: typeof result?.dataSource === 'string' diff --git a/web-console/src/druid-models/stages/stages.ts b/web-console/src/druid-models/stages/stages.ts index 04cf22be8f9..0d263a40553 100644 --- a/web-console/src/druid-models/stages/stages.ts +++ b/web-console/src/druid-models/stages/stages.ts @@ -74,6 +74,8 @@ export interface StageDefinition { startTime?: string; duration?: number; sort?: boolean; + shuffle?: string; + output?: string; } export interface ClusterBy { @@ -169,7 +171,7 @@ export type SegmentGenerationProgressFields = | 'rowsPushed'; export interface WarningCounter { - type: 'warning'; + type: 'warnings'; CannotParseExternalData?: number; // More types of warnings might be added later } @@ -192,6 +194,8 @@ function zeroChannelFields(): Record { }; } +export type Counters = Record>; + export class Stages { static readonly QUERY_START_FACTOR = 0.05; static readonly QUERY_END_FACTOR = 0.05; @@ -205,12 +209,9 @@ export class Stages { } public readonly stages: StageDefinition[]; - private readonly counters?: Record>; + private readonly counters?: Counters; - constructor( - stages: StageDefinition[], - counters?: Record>, - ) { + constructor(stages: StageDefinition[], counters?: Counters) { this.stages = stages; this.counters = counters; } diff --git a/web-console/src/druid-models/task/task.ts b/web-console/src/druid-models/task/task.ts index f8e329c3405..1a245508e8d 100644 --- a/web-console/src/druid-models/task/task.ts +++ b/web-console/src/druid-models/task/task.ts @@ -18,7 +18,7 @@ import { C } from '@druid-toolkit/query'; -import type { StageDefinition } from '../stages/stages'; +import type { Counters, StageDefinition } from '../stages/stages'; export type TaskStatus = 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS'; export type TaskStatusWithCanceled = TaskStatus | 'CANCELED'; @@ -112,7 +112,7 @@ export interface MsqTaskReportResponse { segmentLoadWaiterStatus?: SegmentLoadWaiterStatus; }; stages: StageDefinition[]; - counters: Record>; + counters: Counters; }; }; error?: any; diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 05818b27606..76bd29a4448 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -151,11 +151,12 @@ export class WorkbenchQuery { return WorkbenchQuery.enabledQueryEngines; } - static fromEffectiveQueryAndContext(queryString: string, context: QueryContext): WorkbenchQuery { + static fromTaskQueryAndContext(queryString: string, context: QueryContext): WorkbenchQuery { const noSqlOuterLimit = typeof context['sqlOuterLimit'] === 'undefined'; const cleanContext = deleteKeys(context, ['sqlOuterLimit']); let retQuery = WorkbenchQuery.blank() + .changeEngine('sql-msq-task') .changeQueryString(queryString) .changeQueryContext(cleanContext); diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index ca7c0485bd4..0fa9b090959 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -184,7 +184,7 @@ export async function getTaskExecution( if (!execution) { const statusResp = await Api.instance.get( - `/druid/v2/sql/statements/${encodedId}`, + `/druid/v2/sql/statements/${encodedId}?detail=true`, { cancelToken, }, diff --git a/web-console/src/utils/download.spec.ts b/web-console/src/utils/download.spec.ts index 85d6669acc9..71c051500d0 100644 --- a/web-console/src/utils/download.spec.ts +++ b/web-console/src/utils/download.spec.ts @@ -20,7 +20,10 @@ import { formatForFormat } from './download'; describe('download', () => { it('.formatForFormat', () => { - expect(formatForFormat(null, 'csv')).toEqual('"null"'); + expect(formatForFormat(null, 'csv')).toEqual(''); + expect(formatForFormat(null, 'tsv')).toEqual(''); + expect(formatForFormat('', 'csv')).toEqual('""'); + expect(formatForFormat('null', 'csv')).toEqual('"null"'); expect(formatForFormat('hello\nworld', 'csv')).toEqual('"hello world"'); expect(formatForFormat(123, 'csv')).toEqual('"123"'); expect(formatForFormat(new Date('2021-01-02T03:04:05.678Z'), 'csv')).toEqual( diff --git a/web-console/src/utils/download.ts b/web-console/src/utils/download.ts index aabe2fc1c66..4fb3342e523 100644 --- a/web-console/src/utils/download.ts +++ b/web-console/src/utils/download.ts @@ -43,6 +43,8 @@ export function downloadUrl(url: string, filename: string) { } export function formatForFormat(s: null | string | number | Date, format: 'csv' | 'tsv'): string { + if (s == null) return ''; + // stringify and remove line break const str = stringifyValue(s).replace(/(?:\r\n|\r|\n)/g, ' '); diff --git a/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx b/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx index 47ba4dd7705..4f254d13407 100644 --- a/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx +++ b/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx @@ -173,7 +173,7 @@ LIMIT 100`, } onNewTab( - WorkbenchQuery.fromEffectiveQueryAndContext( + WorkbenchQuery.fromTaskQueryAndContext( execution.sqlQuery, execution.queryContext, ).changeLastExecution({ engine: 'sql-msq-task', id: w.taskId }), diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx index a125b8d2987..aaac54e6d4a 100644 --- a/web-console/src/views/workbench-view/workbench-view.tsx +++ b/web-console/src/views/workbench-view/workbench-view.tsx @@ -30,7 +30,8 @@ import type { SqlQuery } from '@druid-toolkit/query'; import { SqlExpression } from '@druid-toolkit/query'; import classNames from 'classnames'; import copy from 'copy-to-clipboard'; -import React, { ComponentProps } from 'react'; +import type { ComponentProps } from 'react'; +import React from 'react'; import { SpecDialog, StringInputDialog } from '../../dialogs'; import type { @@ -464,7 +465,7 @@ export class WorkbenchView extends React.PureComponent