From 55ed69f8300764ce34a26bdf3fdf365df0099e6e Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 24 Jan 2024 11:41:52 -0800 Subject: [PATCH] update mocks (#15745) --- .../execution-ingest-complete.mock.ts | 72 +- .../execution/execution-ingest-error.mock.ts | 61 +- .../druid-models/execution/execution.spec.ts | 66 +- .../src/druid-models/execution/execution.ts | 25 +- .../src/druid-models/stages/stages.mock.ts | 1006 ++++++++++++----- .../src/druid-models/stages/stages.spec.ts | 474 ++++---- web-console/src/druid-models/stages/stages.ts | 3 + web-console/src/druid-models/task/task.ts | 20 + .../execution-details-pane.spec.tsx.snap | 440 +++---- .../execution-error-pane.spec.tsx.snap | 2 +- .../execution-progress-bar-pane.spec.tsx.snap | 2 +- .../execution-stages-pane.spec.tsx.snap | 18 +- .../execution-stages-pane.tsx | 14 +- .../ingest-success-pane.spec.tsx.snap | 4 +- 14 files changed, 1369 insertions(+), 838 deletions(-) diff --git a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts index 3ae77b27ef2..07d1025001d 100644 --- a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts @@ -43,20 +43,40 @@ PARTITIONED BY ALL TIME export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ multiStageQuery: { type: 'multiStageQuery', - taskId: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + taskId: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', payload: { status: { status: 'SUCCESS', - startTime: '2023-08-01T03:12:59.527Z', - durationMs: 23699, + startTime: '2024-01-23T19:45:43.073Z', + durationMs: 14208, + workers: { + '0': [ + { + workerId: 'query-346b9ac6-4912-46e4-9b98-75f11071af87-worker0_0', + state: 'SUCCESS', + durationMs: 8789, + }, + ], + }, pendingTasks: 0, - runningTasks: 2, + runningTasks: 1, + segmentLoadWaiterStatus: { + state: 'SUCCESS', + startTime: '2024-01-23T19:45:52.189Z', + duration: 5092, + totalSegments: 1, + usedSegments: 1, + precachedSegments: 1, + onDemandSegments: 0, + pendingSegments: 0, + unknownSegments: 0, + }, }, stages: [ { stageNumber: 0, definition: { - id: 'ad318360-2ccf-4afc-b221-27c8704bf4fe_0', + id: '7f62fa91-f49a-4053-adec-5aa09c251ee3_0', input: [ { type: 'external', @@ -127,6 +147,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ columns: ['agent_type', 'v0'], legacy: false, context: { + __resultFormat: 'array', __timeColumn: 'v0', __user: 'allowAll', executionMode: 'async', @@ -135,12 +156,13 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ groupByEnableMultiValueUnnesting: false, maxNumTasks: 2, maxParseExceptions: 0, - queryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + queryId: '346b9ac6-4912-46e4-9b98-75f11071af87', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + sqlQueryId: '346b9ac6-4912-46e4-9b98-75f11071af87', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, granularity: { type: 'all', @@ -179,14 +201,14 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ phase: 'FINISHED', workerCount: 1, partitionCount: 1, - startTime: '2023-08-01T03:12:59.865Z', - duration: 21324, + startTime: '2024-01-23T19:45:43.302Z', + duration: 6884, sort: true, }, { stageNumber: 1, definition: { - id: 'ad318360-2ccf-4afc-b221-27c8704bf4fe_1', + id: '7f62fa91-f49a-4053-adec-5aa09c251ee3_1', input: [ { type: 'stage', @@ -251,8 +273,8 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ phase: 'FINISHED', workerCount: 1, partitionCount: 1, - startTime: '2023-08-01T03:13:21.156Z', - duration: 2070, + startTime: '2024-01-23T19:45:50.170Z', + duration: 1263, }, ], counters: { @@ -317,10 +339,10 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ }, }) .updateWithTaskPayload({ - task: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + task: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', payload: { type: 'query_controller', - id: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + id: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', spec: { query: { queryType: 'scan', @@ -363,6 +385,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ columns: ['agent_type', 'v0'], legacy: false, context: { + __resultFormat: 'array', __user: 'allowAll', executionMode: 'async', finalize: false, @@ -370,11 +393,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ groupByEnableMultiValueUnnesting: false, maxNumTasks: 2, maxParseExceptions: 0, - queryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + queryId: '346b9ac6-4912-46e4-9b98-75f11071af87', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + sqlQueryId: '346b9ac6-4912-46e4-9b98-75f11071af87', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, granularity: { type: 'all', @@ -409,13 +433,15 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME', sqlQueryContext: { finalizeAggregations: false, - sqlQueryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + sqlQueryId: '346b9ac6-4912-46e4-9b98-75f11071af87', groupByEnableMultiValueUnnesting: false, sqlInsertSegmentGranularity: '{"type":"all"}', maxNumTasks: 2, + waitUntilSegmentsLoad: true, sqlReplaceTimeChunks: 'all', executionMode: 'async', - queryId: '93a855fa-c35a-48df-b596-6bc98eed1101', + __resultFormat: 'array', + queryId: '346b9ac6-4912-46e4-9b98-75f11071af87', }, sqlResultsContext: { timeZone: 'UTC', @@ -428,19 +454,19 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true, }, - groupId: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + groupId: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', dataSource: 'kttm_simple', resource: { - availabilityGroup: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + availabilityGroup: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', requiredCapacity: 1, }, }, }) .updateWithAsyncStatus({ - queryId: 'query-93a855fa-c35a-48df-b596-6bc98eed1101', + queryId: 'query-346b9ac6-4912-46e4-9b98-75f11071af87', state: 'SUCCESS', - createdAt: '2023-08-01T03:12:50.121Z', - durationMs: 37657, + createdAt: '2024-01-23T19:45:41.136Z', + durationMs: 16637, result: { numTotalRows: 465346, totalSizeInBytes: 0, diff --git a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts index 9d58a7e558c..3c1d50327db 100644 --- a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts @@ -44,12 +44,12 @@ PARTITIONED BY DAY export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ multiStageQuery: { type: 'multiStageQuery', - taskId: 'query-af8a263d-213f-418e-ad8d-37d55beff59b', + taskId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754', payload: { status: { status: 'FAILED', errorReport: { - taskId: 'query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0', + taskId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0', host: 'localhost', error: { errorCode: 'TooManyWarnings', @@ -60,7 +60,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ }, warnings: [ { - taskId: 'query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0', + taskId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0', host: 'localhost:8101', stageNumber: 0, error: { @@ -69,10 +69,10 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)', }, exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 23 more\n', + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n', }, { - taskId: 'query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0', + taskId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0', host: 'localhost:8101', stageNumber: 0, error: { @@ -81,19 +81,28 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)', }, exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 23 more\n', + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n', }, ], - startTime: '2023-08-01T04:20:24.945Z', - durationMs: 14545, + startTime: '2024-01-23T19:56:44.633Z', + durationMs: 4483, + workers: { + '0': [ + { + workerId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0', + state: 'FAILED', + durationMs: -1, + }, + ], + }, pendingTasks: 0, - runningTasks: 2, + runningTasks: 1, }, stages: [ { stageNumber: 0, definition: { - id: 'f635e36d-6b90-4b74-ad5e-a179c99f0ddc_0', + id: '2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_0', input: [ { type: 'external', @@ -166,6 +175,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ columns: ['agent_type', 'v0'], legacy: false, context: { + __resultFormat: 'array', __timeColumn: 'v0', __user: 'allowAll', executionMode: 'async', @@ -174,12 +184,13 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ groupByEnableMultiValueUnnesting: false, maxNumTasks: 2, maxParseExceptions: 2, - queryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + queryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '"DAY"', - sqlQueryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + sqlQueryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, granularity: { type: 'all', @@ -226,14 +237,14 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ }, phase: 'FAILED', workerCount: 1, - startTime: '2023-08-01T04:20:25.296Z', - duration: 14193, + startTime: '2024-01-23T19:56:44.857Z', + duration: 4259, sort: true, }, { stageNumber: 1, definition: { - id: 'f635e36d-6b90-4b74-ad5e-a179c99f0ddc_1', + id: '2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_1', input: [ { type: 'stage', @@ -335,10 +346,10 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ }, }, }).updateWithTaskPayload({ - task: 'query-af8a263d-213f-418e-ad8d-37d55beff59b', + task: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754', payload: { type: 'query_controller', - id: 'query-af8a263d-213f-418e-ad8d-37d55beff59b', + id: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754', spec: { query: { queryType: 'scan', @@ -381,6 +392,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ columns: ['agent_type', 'v0'], legacy: false, context: { + __resultFormat: 'array', __user: 'allowAll', executionMode: 'async', finalize: false, @@ -388,11 +400,12 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ groupByEnableMultiValueUnnesting: false, maxNumTasks: 2, maxParseExceptions: 2, - queryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + queryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '"DAY"', - sqlQueryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + sqlQueryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, granularity: { type: 'all', @@ -426,13 +439,15 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ sqlQueryContext: { maxParseExceptions: 2, finalizeAggregations: false, - sqlQueryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + sqlQueryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', groupByEnableMultiValueUnnesting: false, sqlInsertSegmentGranularity: '"DAY"', maxNumTasks: 2, + waitUntilSegmentsLoad: true, sqlReplaceTimeChunks: 'all', executionMode: 'async', - queryId: 'af8a263d-213f-418e-ad8d-37d55beff59b', + __resultFormat: 'array', + queryId: '26d490c6-c06d-4cd2-938f-bc5f7f982754', }, sqlResultsContext: { timeZone: 'UTC', @@ -445,10 +460,10 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true, }, - groupId: 'query-af8a263d-213f-418e-ad8d-37d55beff59b', + groupId: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754', dataSource: 'kttm-blank-lines', resource: { - availabilityGroup: 'query-af8a263d-213f-418e-ad8d-37d55beff59b', + availabilityGroup: 'query-26d490c6-c06d-4cd2-938f-bc5f7f982754', requiredCapacity: 1, }, }, diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index fac129baf9b..9b0bf4106db 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -42,14 +42,14 @@ describe('Execution', () => { "useLineageBasedSegmentAllocation": true, }, "dataSource": "kttm_simple", - "groupId": "query-93a855fa-c35a-48df-b596-6bc98eed1101", - "id": "query-93a855fa-c35a-48df-b596-6bc98eed1101", + "groupId": "query-346b9ac6-4912-46e4-9b98-75f11071af87", + "id": "query-346b9ac6-4912-46e4-9b98-75f11071af87", "nativeTypeNames": Array [ "LONG", "STRING", ], "resource": Object { - "availabilityGroup": "query-93a855fa-c35a-48df-b596-6bc98eed1101", + "availabilityGroup": "query-346b9ac6-4912-46e4-9b98-75f11071af87", "requiredCapacity": 1, }, "spec": Object { @@ -80,6 +80,7 @@ describe('Execution', () => { "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -87,11 +88,12 @@ describe('Execution', () => { "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 0, - "queryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "queryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "sqlQueryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -157,14 +159,16 @@ describe('Execution', () => { ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY ALL TIME", "sqlQueryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, - "queryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "queryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "sqlQueryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "sqlResultsContext": Object { "serializeComplexValues": true, @@ -177,7 +181,7 @@ describe('Execution', () => { ], "type": "query_controller", }, - "task": "query-93a855fa-c35a-48df-b596-6bc98eed1101", + "task": "query-346b9ac6-4912-46e4-9b98-75f11071af87", }, "capacityInfo": undefined, "destination": Object { @@ -192,16 +196,17 @@ describe('Execution', () => { "type": "dataSource", }, "destinationPages": undefined, - "duration": 23699, + "duration": 14208, "engine": "sql-msq-task", "error": undefined, - "id": "query-93a855fa-c35a-48df-b596-6bc98eed1101", + "id": "query-346b9ac6-4912-46e4-9b98-75f11071af87", "nativeQuery": Object { "columns": Array [ "agent_type", "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -209,11 +214,12 @@ describe('Execution', () => { "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 0, - "queryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "queryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "sqlQueryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -262,13 +268,25 @@ describe('Execution', () => { ], }, "queryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, + "waitUntilSegmentsLoad": true, }, "result": undefined, - "segmentStatus": undefined, + "segmentStatus": Object { + "duration": 5092, + "onDemandSegments": 0, + "pendingSegments": 0, + "precachedSegments": 1, + "startTime": "2024-01-23T19:45:52.189Z", + "state": "SUCCESS", + "totalSegments": 1, + "unknownSegments": 0, + "usedSegments": 1, + }, "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -368,7 +386,7 @@ describe('Execution', () => { "stages": Array [ Object { "definition": Object { - "id": "ad318360-2ccf-4afc-b221-27c8704bf4fe_0", + "id": "7f62fa91-f49a-4053-adec-5aa09c251ee3_0", "input": Array [ Object { "inputFormat": Object { @@ -404,6 +422,7 @@ describe('Execution', () => { "v0", ], "context": Object { + "__resultFormat": "array", "__timeColumn": "v0", "__user": "allowAll", "executionMode": "async", @@ -412,11 +431,12 @@ describe('Execution', () => { "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 0, - "queryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "queryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "sqlQueryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -494,17 +514,17 @@ describe('Execution', () => { }, ], }, - "duration": 21324, + "duration": 6884, "partitionCount": 1, "phase": "FINISHED", "sort": true, "stageNumber": 0, - "startTime": "2023-08-01T03:12:59.865Z", + "startTime": "2024-01-23T19:45:43.302Z", "workerCount": 1, }, Object { "definition": Object { - "id": "ad318360-2ccf-4afc-b221-27c8704bf4fe_1", + "id": "7f62fa91-f49a-4053-adec-5aa09c251ee3_1", "input": Array [ Object { "stage": 0, @@ -570,20 +590,20 @@ describe('Execution', () => { }, "signature": Array [], }, - "duration": 2070, + "duration": 1263, "partitionCount": 1, "phase": "FINISHED", "stageNumber": 1, - "startTime": "2023-08-01T03:13:21.156Z", + "startTime": "2024-01-23T19:45:50.170Z", "workerCount": 1, }, ], }, - "startTime": 2023-08-01T03:12:59.527Z, + "startTime": 2024-01-23T19:45:43.073Z, "status": "SUCCESS", "usageInfo": Object { "pendingTasks": 0, - "runningTasks": 2, + "runningTasks": 1, }, "warnings": undefined, } diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 3e8b3af6d6b..4e48528855b 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -32,7 +32,12 @@ import type { DruidEngine } from '../druid-engine/druid-engine'; import { validDruidEngine } from '../druid-engine/druid-engine'; import type { QueryContext } from '../query-context/query-context'; import { Stages } from '../stages/stages'; -import type { MsqTaskPayloadResponse, MsqTaskReportResponse, TaskStatus } from '../task/task'; +import type { + MsqTaskPayloadResponse, + MsqTaskReportResponse, + SegmentLoadWaiterStatus, + TaskStatus, +} from '../task/task'; const IGNORE_CONTEXT_KEYS = [ '__asyncIdentity__', @@ -165,18 +170,6 @@ function formatPendingMessage( } } -interface SegmentStatus { - duration: number; - onDemandSegments: number; - pendingSegments: number; - precachedSegments: number; - startTime: Date; - state: 'INIT' | 'WAITING' | 'SUCCESS'; - totalSegments: number; - unknownSegments: number; - usedSegments: number; -} - export interface ExecutionValue { engine: DruidEngine; id: string; @@ -195,7 +188,7 @@ export interface ExecutionValue { warnings?: ExecutionError[]; capacityInfo?: CapacityInfo; _payload?: MsqTaskPayloadResponse; - segmentStatus?: SegmentStatus; + segmentStatus?: SegmentLoadWaiterStatus; } export class Execution { @@ -306,7 +299,7 @@ export class Execution { const startTime = new Date(deepGet(taskReport, 'multiStageQuery.payload.status.startTime')); const durationMs = deepGet(taskReport, 'multiStageQuery.payload.status.durationMs'); - const segmentLoaderStatus = deepGet( + const segmentLoaderStatus: SegmentLoadWaiterStatus = deepGet( taskReport, 'multiStageQuery.payload.status.segmentLoadWaiterStatus', ); @@ -389,7 +382,7 @@ export class Execution { public readonly error?: ExecutionError; public readonly warnings?: ExecutionError[]; public readonly capacityInfo?: CapacityInfo; - public readonly segmentStatus?: SegmentStatus; + public readonly segmentStatus?: SegmentLoadWaiterStatus; public readonly _payload?: { payload: any; task: string }; diff --git a/web-console/src/druid-models/stages/stages.mock.ts b/web-console/src/druid-models/stages/stages.mock.ts index 97f816b972d..d38a29c80d1 100644 --- a/web-console/src/druid-models/stages/stages.mock.ts +++ b/web-console/src/druid-models/stages/stages.mock.ts @@ -18,34 +18,93 @@ import { Stages } from './stages'; +/* +===== Query ===== + +-- This demo has a query that is identical to the previous demo except instead of reading the fact data from an external +-- file, it reads the "kttm_simple" datasource that was created in Demo 1. +-- This shows you that you can mix and match data already stored in Druid with external data and transform as needed. +-- +-- In the next demo you will look at another type of data transformation. + +REPLACE INTO "kttm_reingest" OVERWRITE ALL +WITH +country_lookup AS ( +SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', + '{"type":"tsv","findColumnsFromHeader":true}' + ) +) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) + +SELECT + FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, + session, + agent_category, + agent_type, + browser, + browser_version, + CAST(REGEXP_EXTRACT(browser_version, '^(\d+)') AS BIGINT) AS browser_major, + MV_TO_ARRAY("language") AS "language", + os, + city, + country, + country_lookup.Capital AS capital, + country_lookup.ISO3 AS iso3, + forwarded_for AS ip_address, + + COUNT(*) AS "cnt", + SUM(session_length) AS session_length, + APPROX_COUNT_DISTINCT_DS_HLL(event_type) AS unique_event_types +FROM kttm_simple +LEFT JOIN country_lookup ON country_lookup.Country = kttm_simple.country +WHERE os = 'iOS' +GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 +PARTITIONED BY HOUR +CLUSTERED BY browser, session + +===== Context ===== + +{ + "maxNumTasks": 2 +} + */ + export const STAGES = new Stages( [ { stageNumber: 0, definition: { - id: 'ac4d6506-6e1d-4fd7-96e2-193f46219d4b_0', + id: '73775b03-43e9-41fc-be5a-bb320528b4f5_0', input: [ { type: 'external', inputSource: { type: 'http', uris: ['https://static.imply.io/example-data/lookup/countries.tsv'], - httpAuthenticationUsername: null, - httpAuthenticationPassword: null, }, inputFormat: { type: 'tsv', - columns: [], - listDelimiter: null, delimiter: '\t', findColumnsFromHeader: true, - skipHeaderRows: 0, }, signature: [ - { name: 'Country', type: 'STRING' }, - { name: 'Capital', type: 'STRING' }, - { name: 'ISO3', type: 'STRING' }, - { name: 'ISO2', type: 'STRING' }, + { + name: 'Country', + type: 'STRING', + }, + { + name: 'Capital', + type: 'STRING', + }, + { + name: 'ISO3', + type: 'STRING', + }, + { + name: 'ISO2', + type: 'STRING', + }, ], }, ], @@ -53,7 +112,36 @@ export const STAGES = new Stages( type: 'scan', query: { queryType: 'scan', - dataSource: { type: 'inputNumber', inputNumber: 0 }, + dataSource: { + type: 'external', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/example-data/lookup/countries.tsv'], + }, + inputFormat: { + type: 'tsv', + delimiter: '\t', + findColumnsFromHeader: true, + }, + signature: [ + { + name: 'Country', + type: 'STRING', + }, + { + name: 'Capital', + type: 'STRING', + }, + { + name: 'ISO3', + type: 'STRING', + }, + { + name: 'ISO2', + type: 'STRING', + }, + ], + }, intervals: { type: 'intervals', intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], @@ -62,51 +150,84 @@ export const STAGES = new Stages( columns: ['Capital', 'Country', 'ISO3'], legacy: false, context: { - groupByEnableMultiValueUnnesting: false, + __resultFormat: 'array', + executionMode: 'async', finalizeAggregations: false, - maxNumTasks: 3, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + queryId: '63529263-1b27-4e4b-bc77-aa93647949f1', scanSignature: '[{"name":"Capital","type":"STRING"},{"name":"Country","type":"STRING"},{"name":"ISO3","type":"STRING"}]', - multiStageQuery: true, - sqlInsertSegmentGranularity: '"HOUR"', - sqlQueryId: '57c2dca3-0199-4c61-bef0-344a68ff351b', + sqlInsertSegmentGranularity: null, + sqlQueryId: '63529263-1b27-4e4b-bc77-aa93647949f1', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, + }, + granularity: { + type: 'all', }, - granularity: { type: 'all' }, }, }, signature: [ - { name: '__bucket', type: 'LONG' }, - { name: '__boost', type: 'LONG' }, - { name: 'Capital', type: 'STRING' }, - { name: 'Country', type: 'STRING' }, - { name: 'ISO3', type: 'STRING' }, + { + name: '__boost', + type: 'LONG', + }, + { + name: 'Capital', + type: 'STRING', + }, + { + name: 'Country', + type: 'STRING', + }, + { + name: 'ISO3', + type: 'STRING', + }, ], shuffleSpec: { type: 'maxCount', clusterBy: { - columns: [{ columnName: '__bucket' }, { columnName: '__boost' }], - bucketByCount: 1, + columns: [ + { + columnName: '__boost', + order: 'ASCENDING', + }, + ], }, - partitions: 2, - aggregate: false, + partitions: 1, }, - maxWorkerCount: 2, + maxWorkerCount: 1, }, phase: 'FINISHED', - workerCount: 2, - partitionCount: 2, - startTime: '2022-07-06T16:27:32.835Z', - duration: 313, + workerCount: 1, + partitionCount: 1, + startTime: '2024-01-23T18:58:43.197Z', + duration: 4115, sort: true, }, { stageNumber: 1, definition: { - id: 'ac4d6506-6e1d-4fd7-96e2-193f46219d4b_1', + id: '73775b03-43e9-41fc-be5a-bb320528b4f5_1', input: [ - { type: 'table', dataSource: 'kttm_simple' }, - { type: 'stage', stage: 0 }, + { + type: 'table', + dataSource: 'kttm_simple', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + filter: { + type: 'equals', + column: 'os', + matchValueType: 'STRING', + matchValue: 'iOS', + }, + filterFields: ['os'], + }, + { + type: 'stage', + stage: 0, + }, ], broadcast: [1], processor: { @@ -115,8 +236,14 @@ export const STAGES = new Stages( queryType: 'groupBy', dataSource: { type: 'join', - left: { type: 'inputNumber', inputNumber: 0 }, - right: { type: 'inputNumber', inputNumber: 1 }, + left: { + type: 'inputNumber', + inputNumber: 0, + }, + right: { + type: 'inputNumber', + inputNumber: 1, + }, rightPrefix: 'j0.', condition: '("country" == "j0.Country")', joinType: 'LEFT', @@ -139,12 +266,29 @@ export const STAGES = new Stages( expression: 'mv_to_array("language")', outputType: 'ARRAY', }, - { type: 'expression', name: 'v2', expression: "'iOS'", outputType: 'STRING' }, + { + type: 'expression', + name: 'v2', + expression: "'iOS'", + outputType: 'STRING', + }, ], - filter: { type: 'selector', dimension: 'os', value: 'iOS' }, - granularity: { type: 'all' }, + filter: { + type: 'equals', + column: 'os', + matchValueType: 'STRING', + matchValue: 'iOS', + }, + granularity: { + type: 'all', + }, dimensions: [ - { type: 'default', dimension: 'v0', outputName: 'd0', outputType: 'LONG' }, + { + type: 'default', + dimension: 'v0', + outputName: 'd0', + outputType: 'LONG', + }, { type: 'default', dimension: 'session', @@ -194,7 +338,12 @@ export const STAGES = new Stages( outputName: 'd7', outputType: 'ARRAY', }, - { type: 'default', dimension: 'v2', outputName: 'd8', outputType: 'STRING' }, + { + type: 'default', + dimension: 'v2', + outputName: 'd8', + outputType: 'STRING', + }, { type: 'default', dimension: 'city', @@ -227,8 +376,15 @@ export const STAGES = new Stages( }, ], aggregations: [ - { type: 'count', name: 'a0' }, - { type: 'longSum', name: 'a1', fieldName: 'session_length' }, + { + type: 'count', + name: 'a0', + }, + { + type: 'longSum', + name: 'a1', + fieldName: 'session_length', + }, { type: 'HLLSketchBuild', name: 'a2', @@ -244,94 +400,205 @@ export const STAGES = new Stages( { dimension: 'd4', direction: 'ascending', - dimensionOrder: { type: 'lexicographic' }, + dimensionOrder: { + type: 'lexicographic', + }, }, { dimension: 'd1', direction: 'ascending', - dimensionOrder: { type: 'lexicographic' }, + dimensionOrder: { + type: 'lexicographic', + }, }, ], }, context: { + __resultFormat: 'array', + __timeColumn: 'd0', + __user: 'allowAll', + executionMode: 'async', finalize: false, - groupByEnableMultiValueUnnesting: false, finalizeAggregations: false, - maxNumTasks: 3, - scanSignature: - '[{"name":"Capital","type":"STRING"},{"name":"Country","type":"STRING"},{"name":"ISO3","type":"STRING"}]', - msqTimeColumn: 'd0', - multiStageQuery: true, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '63529263-1b27-4e4b-bc77-aa93647949f1', sqlInsertSegmentGranularity: '"HOUR"', - sqlQueryId: '57c2dca3-0199-4c61-bef0-344a68ff351b', + sqlQueryId: '63529263-1b27-4e4b-bc77-aa93647949f1', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, }, }, signature: [ - { name: 'd0', type: 'LONG' }, - { name: 'd1', type: 'STRING' }, - { name: 'd2', type: 'STRING' }, - { name: 'd3', type: 'STRING' }, - { name: 'd4', type: 'STRING' }, - { name: 'd5', type: 'STRING' }, - { name: 'd6', type: 'LONG' }, - { name: 'd7', type: 'ARRAY' }, - { name: 'd8', type: 'STRING' }, - { name: 'd9', type: 'STRING' }, - { name: 'd10', type: 'STRING' }, - { name: 'd11', type: 'STRING' }, - { name: 'd12', type: 'STRING' }, - { name: 'd13', type: 'STRING' }, - { name: 'a0', type: 'LONG' }, - { name: 'a1', type: 'LONG' }, - { name: 'a2', type: 'COMPLEX' }, + { + name: 'd0', + type: 'LONG', + }, + { + name: 'd1', + type: 'STRING', + }, + { + name: 'd2', + type: 'STRING', + }, + { + name: 'd3', + type: 'STRING', + }, + { + name: 'd4', + type: 'STRING', + }, + { + name: 'd5', + type: 'STRING', + }, + { + name: 'd6', + type: 'LONG', + }, + { + name: 'd7', + type: 'ARRAY', + }, + { + name: 'd8', + type: 'STRING', + }, + { + name: 'd9', + type: 'STRING', + }, + { + name: 'd10', + type: 'STRING', + }, + { + name: 'd11', + type: 'STRING', + }, + { + name: 'd12', + type: 'STRING', + }, + { + name: 'd13', + type: 'STRING', + }, + { + name: 'a0', + type: 'LONG', + }, + { + name: 'a1', + type: 'LONG', + }, + { + name: 'a2', + type: 'COMPLEX', + }, ], shuffleSpec: { type: 'maxCount', clusterBy: { columns: [ - { columnName: 'd0' }, - { columnName: 'd1' }, - { columnName: 'd2' }, - { columnName: 'd3' }, - { columnName: 'd4' }, - { columnName: 'd5' }, - { columnName: 'd6' }, - { columnName: 'd7' }, - { columnName: 'd8' }, - { columnName: 'd9' }, - { columnName: 'd10' }, - { columnName: 'd11' }, - { columnName: 'd12' }, - { columnName: 'd13' }, + { + columnName: 'd0', + order: 'ASCENDING', + }, + { + columnName: 'd1', + order: 'ASCENDING', + }, + { + columnName: 'd2', + order: 'ASCENDING', + }, + { + columnName: 'd3', + order: 'ASCENDING', + }, + { + columnName: 'd4', + order: 'ASCENDING', + }, + { + columnName: 'd5', + order: 'ASCENDING', + }, + { + columnName: 'd6', + order: 'ASCENDING', + }, + { + columnName: 'd7', + order: 'ASCENDING', + }, + { + columnName: 'd8', + order: 'ASCENDING', + }, + { + columnName: 'd9', + order: 'ASCENDING', + }, + { + columnName: 'd10', + order: 'ASCENDING', + }, + { + columnName: 'd11', + order: 'ASCENDING', + }, + { + columnName: 'd12', + order: 'ASCENDING', + }, + { + columnName: 'd13', + order: 'ASCENDING', + }, ], }, - partitions: 2, + partitions: 1, aggregate: true, }, - maxWorkerCount: 2, + maxWorkerCount: 1, }, phase: 'FINISHED', - workerCount: 2, - partitionCount: 2, - startTime: '2022-07-06T16:27:33.139Z', - duration: 1927, + workerCount: 1, + partitionCount: 1, + startTime: '2024-01-23T18:58:47.300Z', + duration: 911, sort: true, }, { stageNumber: 2, definition: { - id: 'ac4d6506-6e1d-4fd7-96e2-193f46219d4b_2', - input: [{ type: 'stage', stage: 1 }], + id: '73775b03-43e9-41fc-be5a-bb320528b4f5_2', + input: [ + { + type: 'stage', + stage: 1, + }, + ], processor: { type: 'groupByPostShuffle', query: { queryType: 'groupBy', dataSource: { type: 'join', - left: { type: 'inputNumber', inputNumber: 0 }, - right: { type: 'inputNumber', inputNumber: 1 }, + left: { + type: 'inputNumber', + inputNumber: 0, + }, + right: { + type: 'inputNumber', + inputNumber: 1, + }, rightPrefix: 'j0.', condition: '("country" == "j0.Country")', joinType: 'LEFT', @@ -354,12 +621,29 @@ export const STAGES = new Stages( expression: 'mv_to_array("language")', outputType: 'ARRAY', }, - { type: 'expression', name: 'v2', expression: "'iOS'", outputType: 'STRING' }, + { + type: 'expression', + name: 'v2', + expression: "'iOS'", + outputType: 'STRING', + }, ], - filter: { type: 'selector', dimension: 'os', value: 'iOS' }, - granularity: { type: 'all' }, + filter: { + type: 'equals', + column: 'os', + matchValueType: 'STRING', + matchValue: 'iOS', + }, + granularity: { + type: 'all', + }, dimensions: [ - { type: 'default', dimension: 'v0', outputName: 'd0', outputType: 'LONG' }, + { + type: 'default', + dimension: 'v0', + outputName: 'd0', + outputType: 'LONG', + }, { type: 'default', dimension: 'session', @@ -409,7 +693,12 @@ export const STAGES = new Stages( outputName: 'd7', outputType: 'ARRAY', }, - { type: 'default', dimension: 'v2', outputName: 'd8', outputType: 'STRING' }, + { + type: 'default', + dimension: 'v2', + outputName: 'd8', + outputType: 'STRING', + }, { type: 'default', dimension: 'city', @@ -442,8 +731,15 @@ export const STAGES = new Stages( }, ], aggregations: [ - { type: 'count', name: 'a0' }, - { type: 'longSum', name: 'a1', fieldName: 'session_length' }, + { + type: 'count', + name: 'a0', + }, + { + type: 'longSum', + name: 'a1', + fieldName: 'session_length', + }, { type: 'HLLSketchBuild', name: 'a2', @@ -459,79 +755,161 @@ export const STAGES = new Stages( { dimension: 'd4', direction: 'ascending', - dimensionOrder: { type: 'lexicographic' }, + dimensionOrder: { + type: 'lexicographic', + }, }, { dimension: 'd1', direction: 'ascending', - dimensionOrder: { type: 'lexicographic' }, + dimensionOrder: { + type: 'lexicographic', + }, }, ], }, context: { + __resultFormat: 'array', + __timeColumn: 'd0', + __user: 'allowAll', + executionMode: 'async', finalize: false, - groupByEnableMultiValueUnnesting: false, finalizeAggregations: false, - maxNumTasks: 3, - scanSignature: - '[{"name":"Capital","type":"STRING"},{"name":"Country","type":"STRING"},{"name":"ISO3","type":"STRING"}]', - msqTimeColumn: 'd0', - multiStageQuery: true, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '63529263-1b27-4e4b-bc77-aa93647949f1', sqlInsertSegmentGranularity: '"HOUR"', - sqlQueryId: '57c2dca3-0199-4c61-bef0-344a68ff351b', + sqlQueryId: '63529263-1b27-4e4b-bc77-aa93647949f1', sqlReplaceTimeChunks: 'all', + waitUntilSegmentsLoad: true, }, }, }, signature: [ - { name: '__bucket', type: 'LONG' }, - { name: 'd4', type: 'STRING' }, - { name: 'd1', type: 'STRING' }, - { name: 'd0', type: 'LONG' }, - { name: 'd2', type: 'STRING' }, - { name: 'd3', type: 'STRING' }, - { name: 'd5', type: 'STRING' }, - { name: 'd6', type: 'LONG' }, - { name: 'd7', type: 'ARRAY' }, - { name: 'd8', type: 'STRING' }, - { name: 'd9', type: 'STRING' }, - { name: 'd10', type: 'STRING' }, - { name: 'd11', type: 'STRING' }, - { name: 'd12', type: 'STRING' }, - { name: 'd13', type: 'STRING' }, - { name: 'a0', type: 'LONG' }, - { name: 'a1', type: 'LONG' }, - { name: 'a2', type: 'COMPLEX' }, + { + name: '__bucket', + type: 'LONG', + }, + { + name: 'd4', + type: 'STRING', + }, + { + name: 'd1', + type: 'STRING', + }, + { + name: 'd0', + type: 'LONG', + }, + { + name: 'd2', + type: 'STRING', + }, + { + name: 'd3', + type: 'STRING', + }, + { + name: 'd5', + type: 'STRING', + }, + { + name: 'd6', + type: 'LONG', + }, + { + name: 'd7', + type: 'ARRAY', + }, + { + name: 'd8', + type: 'STRING', + }, + { + name: 'd9', + type: 'STRING', + }, + { + name: 'd10', + type: 'STRING', + }, + { + name: 'd11', + type: 'STRING', + }, + { + name: 'd12', + type: 'STRING', + }, + { + name: 'd13', + type: 'STRING', + }, + { + name: 'a0', + type: 'LONG', + }, + { + name: 'a1', + type: 'LONG', + }, + { + name: 'a2', + type: 'COMPLEX', + }, ], shuffleSpec: { type: 'targetSize', clusterBy: { - columns: [{ columnName: '__bucket' }, { columnName: 'd4' }, { columnName: 'd1' }], + columns: [ + { + columnName: '__bucket', + order: 'ASCENDING', + }, + { + columnName: 'd4', + order: 'ASCENDING', + }, + { + columnName: 'd1', + order: 'ASCENDING', + }, + ], bucketByCount: 1, }, targetSize: 3000000, - aggregate: false, }, - maxWorkerCount: 2, + maxWorkerCount: 1, shuffleCheckHasMultipleValues: true, }, phase: 'FINISHED', - workerCount: 2, + workerCount: 1, partitionCount: 24, - startTime: '2022-07-06T16:27:35.052Z', - duration: 509, + startTime: '2024-01-23T18:58:48.203Z', + duration: 522, sort: true, }, { stageNumber: 3, definition: { - id: 'ac4d6506-6e1d-4fd7-96e2-193f46219d4b_3', - input: [{ type: 'stage', stage: 2 }], + id: '73775b03-43e9-41fc-be5a-bb320528b4f5_3', + input: [ + { + type: 'stage', + stage: 2, + }, + ], processor: { type: 'segmentGenerator', dataSchema: { dataSource: 'kttm_reingest', - timestampSpec: { column: '__time', format: 'millis', missingValue: null }, + timestampSpec: { + column: '__time', + format: 'millis', + missingValue: null, + }, dimensionsSpec: { dimensions: [ { @@ -615,10 +993,19 @@ export const STAGES = new Stages( ], dimensionExclusions: ['__time', 'unique_event_types', 'cnt', 'session_length'], includeAllDimensions: false, + useSchemaDiscovery: false, }, metricsSpec: [ - { type: 'longSum', name: 'cnt', fieldName: 'cnt' }, - { type: 'longSum', name: 'session_length', fieldName: 'session_length' }, + { + type: 'longSum', + name: 'cnt', + fieldName: 'cnt', + }, + { + type: 'longSum', + name: 'session_length', + fieldName: 'session_length', + }, { type: 'HLLSketchMerge', name: 'unique_event_types', @@ -630,115 +1017,140 @@ export const STAGES = new Stages( ], granularitySpec: { type: 'arbitrary', - queryGranularity: { type: 'none' }, + queryGranularity: { + type: 'none', + }, rollup: true, intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], }, - transformSpec: { filter: null, transforms: [] }, + transformSpec: { + filter: null, + transforms: [], + }, }, columnMappings: [ - { queryColumn: 'd0', outputColumn: '__time' }, - { queryColumn: 'd1', outputColumn: 'session' }, - { queryColumn: 'd2', outputColumn: 'agent_category' }, - { queryColumn: 'd3', outputColumn: 'agent_type' }, - { queryColumn: 'd4', outputColumn: 'browser' }, - { queryColumn: 'd5', outputColumn: 'browser_version' }, - { queryColumn: 'd6', outputColumn: 'browser_major' }, - { queryColumn: 'd7', outputColumn: 'language' }, - { queryColumn: 'd8', outputColumn: 'os' }, - { queryColumn: 'd9', outputColumn: 'city' }, - { queryColumn: 'd10', outputColumn: 'country' }, - { queryColumn: 'd11', outputColumn: 'capital' }, - { queryColumn: 'd12', outputColumn: 'iso3' }, - { queryColumn: 'd13', outputColumn: 'ip_address' }, - { queryColumn: 'a0', outputColumn: 'cnt' }, - { queryColumn: 'a1', outputColumn: 'session_length' }, - { queryColumn: 'a2', outputColumn: 'unique_event_types' }, + { + queryColumn: 'd0', + outputColumn: '__time', + }, + { + queryColumn: 'd1', + outputColumn: 'session', + }, + { + queryColumn: 'd2', + outputColumn: 'agent_category', + }, + { + queryColumn: 'd3', + outputColumn: 'agent_type', + }, + { + queryColumn: 'd4', + outputColumn: 'browser', + }, + { + queryColumn: 'd5', + outputColumn: 'browser_version', + }, + { + queryColumn: 'd6', + outputColumn: 'browser_major', + }, + { + queryColumn: 'd7', + outputColumn: 'language', + }, + { + queryColumn: 'd8', + outputColumn: 'os', + }, + { + queryColumn: 'd9', + outputColumn: 'city', + }, + { + queryColumn: 'd10', + outputColumn: 'country', + }, + { + queryColumn: 'd11', + outputColumn: 'capital', + }, + { + queryColumn: 'd12', + outputColumn: 'iso3', + }, + { + queryColumn: 'd13', + outputColumn: 'ip_address', + }, + { + queryColumn: 'a0', + outputColumn: 'cnt', + }, + { + queryColumn: 'a1', + outputColumn: 'session_length', + }, + { + queryColumn: 'a2', + outputColumn: 'unique_event_types', + }, ], tuningConfig: { - type: 'index_parallel', - maxRowsPerSegment: 3000000, - appendableIndexSpec: { type: 'onheap', preserveExistingMetrics: false }, + maxNumWorkers: 1, maxRowsInMemory: 100000, - maxBytesInMemory: 0, - skipBytesInMemoryOverheadCheck: false, - maxTotalRows: null, - numShards: null, - splitHintSpec: null, - partitionsSpec: { - type: 'dynamic', - maxRowsPerSegment: 3000000, - maxTotalRows: null, - }, - indexSpec: { - bitmap: { type: 'roaring' }, - dimensionCompression: 'lz4', - metricCompression: 'lz4', - longEncoding: 'longs', - segmentLoader: null, - }, - indexSpecForIntermediatePersists: { - bitmap: { type: 'roaring' }, - dimensionCompression: 'lz4', - metricCompression: 'lz4', - longEncoding: 'longs', - segmentLoader: null, - }, - maxPendingPersists: 0, - forceGuaranteedRollup: false, - reportParseExceptions: false, - pushTimeout: 0, - segmentWriteOutMediumFactory: null, - maxNumConcurrentSubTasks: 2, - maxRetry: 1, - taskStatusCheckPeriodMs: 1000, - chatHandlerTimeout: 'PT10S', - chatHandlerNumRetries: 5, - maxNumSegmentsToMerge: 100, - totalNumMergeTasks: 10, - logParseExceptions: false, - maxParseExceptions: 2147483647, - maxSavedParseExceptions: 0, - maxColumnsToMerge: -1, - awaitSegmentAvailabilityTimeoutMillis: 0, - maxAllowedLockCount: -1, - partitionDimensions: [], + rowsPerSegment: 3000000, }, }, signature: [], - maxWorkerCount: 2, + maxWorkerCount: 1, }, phase: 'FINISHED', - workerCount: 2, + workerCount: 1, partitionCount: 24, - startTime: '2022-07-06T16:27:35.552Z', - duration: 2812, + startTime: '2024-01-23T18:58:48.711Z', + duration: 1669, }, ], { '0': { '0': { - input0: { type: 'channel', rows: [197], files: [1], totalFiles: [1] }, - output: { type: 'channel', rows: [197], bytes: [15468], frames: [1] }, - shuffle: { type: 'channel', rows: [98, 99], bytes: [7260, 7454], frames: [1, 1] }, + input0: { + type: 'channel', + rows: [197], + bytes: [5018], + files: [1], + totalFiles: [1], + }, + output: { + type: 'channel', + rows: [197], + bytes: [12907], + frames: [1], + }, + shuffle: { + type: 'channel', + rows: [197], + bytes: [12119], + frames: [1], + }, sortProgress: { type: 'sortProgress', totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': 2 }, - levelToMergedBatches: { '0': 1, '1': 1, '2': 2 }, - totalMergersForUltimateLevel: 2, - progressDigest: 1.0, - }, - }, - '1': { - sortProgress: { - type: 'sortProgress', - totalMergingLevels: -1, - levelToTotalBatches: {}, - levelToMergedBatches: {}, - totalMergersForUltimateLevel: -1, - triviallyComplete: true, - progressDigest: 1.0, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + totalMergersForUltimateLevel: 1, + progressDigest: 1, }, }, }, @@ -746,87 +1158,89 @@ export const STAGES = new Stages( '0': { input0: { type: 'channel', - rows: [462096], - bytes: [43452909], + rows: [465346], + bytes: [58995494], files: [1], totalFiles: [1], }, - input1: { type: 'channel', rows: [98, 99], bytes: [7260, 7454], frames: [1, 1] }, - output: { type: 'channel', rows: [39667], bytes: [11081283], frames: [2] }, + input1: { + type: 'channel', + rows: [197], + bytes: [12119], + frames: [1], + }, + output: { + type: 'channel', + rows: [39742], + bytes: [11101944], + frames: [2], + }, shuffle: { type: 'channel', - rows: [19834, 19833], - bytes: [5458795, 5464500], - frames: [11, 11], + rows: [39742], + bytes: [10943622], + frames: [21], }, sortProgress: { type: 'sortProgress', totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': 2 }, - levelToMergedBatches: { '0': 1, '1': 1, '2': 2 }, - totalMergersForUltimateLevel: 2, - progressDigest: 1.0, - }, - }, - '1': { - sortProgress: { - type: 'sortProgress', - totalMergingLevels: -1, - levelToTotalBatches: {}, - levelToMergedBatches: {}, - totalMergersForUltimateLevel: -1, - triviallyComplete: true, - progressDigest: 1.0, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, + }, + totalMergersForUltimateLevel: 1, + progressDigest: 1, }, }, }, '2': { '0': { - input0: { type: 'channel', rows: [19834], bytes: [5458795], frames: [11] }, - output: { type: 'channel', rows: [19834], bytes: [5795633], frames: [1] }, + input0: { + type: 'channel', + rows: [39742], + bytes: [10943622], + frames: [21], + }, + output: { + type: 'channel', + rows: [39725], + bytes: [11613687], + frames: [2], + }, shuffle: { type: 'channel', rows: [ - 888, 993, 1418, 894, 590, 633, 309, 241, 236, 309, 253, 260, 440, 876, 1394, 892, 3595, - 5613, + 888, 995, 1419, 905, 590, 652, 322, 247, 236, 309, 256, 260, 440, 876, 1394, 892, 3595, + 6522, 4525, 4326, 4149, 2561, 1914, 1452, ], bytes: [ - 257524, 289161, 412116, 259346, 170554, 182872, 88525, 67821, 65844, 85875, 71002, - 72512, 123204, 249217, 399583, 256916, 1039927, 1625012, + 257524, 289731, 412396, 262388, 170554, 188324, 92275, 69531, 65844, 85875, 71852, + 72512, 123204, 249217, 399583, 256916, 1039927, 1887893, 1307287, 1248166, 1195593, + 738804, 552485, 418062, ], - frames: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4], + frames: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4, 3, 3, 3, 2, 2, 1], }, sortProgress: { type: 'sortProgress', totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': 24 }, - levelToMergedBatches: { '0': 1, '1': 1, '2': 24 }, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 24, + }, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 24, + }, totalMergersForUltimateLevel: 24, - progressDigest: 1.0, - }, - }, - '1': { - input0: { type: 'channel', rows: [0, 19833], bytes: [0, 5464500], frames: [0, 11] }, - output: { type: 'channel', rows: [0, 19833], bytes: [0, 5801321], frames: [0, 1] }, - shuffle: { - type: 'channel', - rows: [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 4525, 4324, 4149, 2561, 1914, - 1451, - ], - bytes: [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262915, 1307287, 1247597, 1195593, - 738804, 552485, 417784, - ], - frames: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 3, 2, 2, 1], - }, - sortProgress: { - type: 'sortProgress', - totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': 24 }, - levelToMergedBatches: { '0': 1, '1': 1, '2': 24 }, - totalMergersForUltimateLevel: 24, - progressDigest: 1.0, + progressDigest: 1, }, }, }, @@ -835,28 +1249,22 @@ export const STAGES = new Stages( input0: { type: 'channel', rows: [ - 888, 0, 1418, 0, 590, 0, 309, 0, 236, 0, 253, 0, 440, 0, 1394, 0, 3595, 0, 4525, 0, - 4149, 0, 1914, + 888, 995, 1419, 905, 590, 652, 322, 247, 236, 309, 256, 260, 440, 876, 1394, 892, 3595, + 6522, 4525, 4326, 4149, 2561, 1914, 1452, ], bytes: [ - 257524, 0, 412116, 0, 170554, 0, 88525, 0, 65844, 0, 71002, 0, 123204, 0, 399583, 0, - 1039927, 0, 1307287, 0, 1195593, 0, 552485, + 257524, 289731, 412396, 262388, 170554, 188324, 92275, 69531, 65844, 85875, 71852, + 72512, 123204, 249217, 399583, 256916, 1039927, 1887893, 1307287, 1248166, 1195593, + 738804, 552485, 418062, ], - frames: [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 2, 0, 3, 0, 3, 0, 2], + frames: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4, 3, 3, 3, 2, 2, 1], }, - }, - '1': { - input0: { - type: 'channel', - rows: [ - 0, 993, 0, 894, 0, 633, 0, 241, 0, 309, 0, 260, 0, 876, 0, 892, 0, 6522, 0, 4324, 0, - 2561, 0, 1451, - ], - bytes: [ - 0, 289161, 0, 259346, 0, 182872, 0, 67821, 0, 85875, 0, 72512, 0, 249217, 0, 256916, 0, - 1887893, 0, 1247597, 0, 738804, 0, 417784, - ], - frames: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 4, 0, 3, 0, 2, 0, 1], + segmentGenerationProgress: { + type: 'segmentGenerationProgress', + rowsProcessed: 39725, + rowsPersisted: 39725, + rowsMerged: 39725, + rowsPushed: 39725, }, }, }, diff --git a/web-console/src/druid-models/stages/stages.spec.ts b/web-console/src/druid-models/stages/stages.spec.ts index 18ffe25b557..c0216fd5c81 100644 --- a/web-console/src/druid-models/stages/stages.spec.ts +++ b/web-console/src/druid-models/stages/stages.spec.ts @@ -27,273 +27,269 @@ describe('Stages', () => { describe('#getByPartitionCountersForStage', () => { it('works for input', () => { - expect(STAGES.getByPartitionCountersForStage(STAGES.stages[2], 'input')).toEqual([ - { - index: 0, - input0: { - bytes: 5458795, - files: 0, - frames: 11, - rows: 19834, - totalFiles: 0, + expect(STAGES.getByPartitionCountersForStage(STAGES.stages[2], 'input')) + .toMatchInlineSnapshot(` + Array [ + Object { + "index": 0, + "input0": Object { + "bytes": 10943622, + "files": 0, + "frames": 21, + "rows": 39742, + "totalFiles": 0, + }, }, - }, - { - index: 1, - input0: { - bytes: 5464500, - files: 0, - frames: 11, - rows: 19833, - totalFiles: 0, - }, - }, - ]); + ] + `); }); it('works for output', () => { - expect(STAGES.getByPartitionCountersForStage(STAGES.stages[2], 'output')).toEqual([ - { - index: 0, - shuffle: { - bytes: 257524, - files: 0, - frames: 1, - rows: 888, - totalFiles: 0, + expect(STAGES.getByPartitionCountersForStage(STAGES.stages[2], 'output')) + .toMatchInlineSnapshot(` + Array [ + Object { + "index": 0, + "shuffle": Object { + "bytes": 257524, + "files": 0, + "frames": 1, + "rows": 888, + "totalFiles": 0, + }, }, - }, - { - index: 1, - shuffle: { - bytes: 289161, - files: 0, - frames: 1, - rows: 993, - totalFiles: 0, + Object { + "index": 1, + "shuffle": Object { + "bytes": 289731, + "files": 0, + "frames": 1, + "rows": 995, + "totalFiles": 0, + }, }, - }, - { - index: 2, - shuffle: { - bytes: 412116, - files: 0, - frames: 1, - rows: 1418, - totalFiles: 0, + Object { + "index": 2, + "shuffle": Object { + "bytes": 412396, + "files": 0, + "frames": 1, + "rows": 1419, + "totalFiles": 0, + }, }, - }, - { - index: 3, - shuffle: { - bytes: 259346, - files: 0, - frames: 1, - rows: 894, - totalFiles: 0, + Object { + "index": 3, + "shuffle": Object { + "bytes": 262388, + "files": 0, + "frames": 1, + "rows": 905, + "totalFiles": 0, + }, }, - }, - { - index: 4, - shuffle: { - bytes: 170554, - files: 0, - frames: 1, - rows: 590, - totalFiles: 0, + Object { + "index": 4, + "shuffle": Object { + "bytes": 170554, + "files": 0, + "frames": 1, + "rows": 590, + "totalFiles": 0, + }, }, - }, - { - index: 5, - shuffle: { - bytes: 182872, - files: 0, - frames: 1, - rows: 633, - totalFiles: 0, + Object { + "index": 5, + "shuffle": Object { + "bytes": 188324, + "files": 0, + "frames": 1, + "rows": 652, + "totalFiles": 0, + }, }, - }, - { - index: 6, - shuffle: { - bytes: 88525, - files: 0, - frames: 1, - rows: 309, - totalFiles: 0, + Object { + "index": 6, + "shuffle": Object { + "bytes": 92275, + "files": 0, + "frames": 1, + "rows": 322, + "totalFiles": 0, + }, }, - }, - { - index: 7, - shuffle: { - bytes: 67821, - files: 0, - frames: 1, - rows: 241, - totalFiles: 0, + Object { + "index": 7, + "shuffle": Object { + "bytes": 69531, + "files": 0, + "frames": 1, + "rows": 247, + "totalFiles": 0, + }, }, - }, - { - index: 8, - shuffle: { - bytes: 65844, - files: 0, - frames: 1, - rows: 236, - totalFiles: 0, + Object { + "index": 8, + "shuffle": Object { + "bytes": 65844, + "files": 0, + "frames": 1, + "rows": 236, + "totalFiles": 0, + }, }, - }, - { - index: 9, - shuffle: { - bytes: 85875, - files: 0, - frames: 1, - rows: 309, - totalFiles: 0, + Object { + "index": 9, + "shuffle": Object { + "bytes": 85875, + "files": 0, + "frames": 1, + "rows": 309, + "totalFiles": 0, + }, }, - }, - { - index: 10, - shuffle: { - bytes: 71002, - files: 0, - frames: 1, - rows: 253, - totalFiles: 0, + Object { + "index": 10, + "shuffle": Object { + "bytes": 71852, + "files": 0, + "frames": 1, + "rows": 256, + "totalFiles": 0, + }, }, - }, - { - index: 11, - shuffle: { - bytes: 72512, - files: 0, - frames: 1, - rows: 260, - totalFiles: 0, + Object { + "index": 11, + "shuffle": Object { + "bytes": 72512, + "files": 0, + "frames": 1, + "rows": 260, + "totalFiles": 0, + }, }, - }, - { - index: 12, - shuffle: { - bytes: 123204, - files: 0, - frames: 1, - rows: 440, - totalFiles: 0, + Object { + "index": 12, + "shuffle": Object { + "bytes": 123204, + "files": 0, + "frames": 1, + "rows": 440, + "totalFiles": 0, + }, }, - }, - { - index: 13, - shuffle: { - bytes: 249217, - files: 0, - frames: 1, - rows: 876, - totalFiles: 0, + Object { + "index": 13, + "shuffle": Object { + "bytes": 249217, + "files": 0, + "frames": 1, + "rows": 876, + "totalFiles": 0, + }, }, - }, - { - index: 14, - shuffle: { - bytes: 399583, - files: 0, - frames: 1, - rows: 1394, - totalFiles: 0, + Object { + "index": 14, + "shuffle": Object { + "bytes": 399583, + "files": 0, + "frames": 1, + "rows": 1394, + "totalFiles": 0, + }, }, - }, - { - index: 15, - shuffle: { - bytes: 256916, - files: 0, - frames: 1, - rows: 892, - totalFiles: 0, + Object { + "index": 15, + "shuffle": Object { + "bytes": 256916, + "files": 0, + "frames": 1, + "rows": 892, + "totalFiles": 0, + }, }, - }, - { - index: 16, - shuffle: { - bytes: 1039927, - files: 0, - frames: 2, - rows: 3595, - totalFiles: 0, + Object { + "index": 16, + "shuffle": Object { + "bytes": 1039927, + "files": 0, + "frames": 2, + "rows": 3595, + "totalFiles": 0, + }, }, - }, - { - index: 17, - shuffle: { - bytes: 1887927, - files: 0, - frames: 5, - rows: 6522, - totalFiles: 0, + Object { + "index": 17, + "shuffle": Object { + "bytes": 1887893, + "files": 0, + "frames": 4, + "rows": 6522, + "totalFiles": 0, + }, }, - }, - { - index: 18, - shuffle: { - bytes: 1307287, - files: 0, - frames: 3, - rows: 4525, - totalFiles: 0, + Object { + "index": 18, + "shuffle": Object { + "bytes": 1307287, + "files": 0, + "frames": 3, + "rows": 4525, + "totalFiles": 0, + }, }, - }, - { - index: 19, - shuffle: { - bytes: 1247597, - files: 0, - frames: 3, - rows: 4324, - totalFiles: 0, + Object { + "index": 19, + "shuffle": Object { + "bytes": 1248166, + "files": 0, + "frames": 3, + "rows": 4326, + "totalFiles": 0, + }, }, - }, - { - index: 20, - shuffle: { - bytes: 1195593, - files: 0, - frames: 3, - rows: 4149, - totalFiles: 0, + Object { + "index": 20, + "shuffle": Object { + "bytes": 1195593, + "files": 0, + "frames": 3, + "rows": 4149, + "totalFiles": 0, + }, }, - }, - { - index: 21, - shuffle: { - bytes: 738804, - files: 0, - frames: 2, - rows: 2561, - totalFiles: 0, + Object { + "index": 21, + "shuffle": Object { + "bytes": 738804, + "files": 0, + "frames": 2, + "rows": 2561, + "totalFiles": 0, + }, }, - }, - { - index: 22, - shuffle: { - bytes: 552485, - files: 0, - frames: 2, - rows: 1914, - totalFiles: 0, + Object { + "index": 22, + "shuffle": Object { + "bytes": 552485, + "files": 0, + "frames": 2, + "rows": 1914, + "totalFiles": 0, + }, }, - }, - { - index: 23, - shuffle: { - bytes: 417784, - files: 0, - frames: 1, - rows: 1451, - totalFiles: 0, + Object { + "index": 23, + "shuffle": Object { + "bytes": 418062, + "files": 0, + "frames": 1, + "rows": 1452, + "totalFiles": 0, + }, }, - }, - ]); + ] + `); }); }); }); diff --git a/web-console/src/druid-models/stages/stages.ts b/web-console/src/druid-models/stages/stages.ts index dd10a83f7da..7cf41a8f464 100644 --- a/web-console/src/druid-models/stages/stages.ts +++ b/web-console/src/druid-models/stages/stages.ts @@ -33,6 +33,9 @@ export type StageInput = | { type: 'table'; dataSource: string; + intervals: string[]; + filter?: any; + filterFields?: string[]; } | { type: 'external'; diff --git a/web-console/src/druid-models/task/task.ts b/web-console/src/druid-models/task/task.ts index 8f6f456ccb4..f8e329c3405 100644 --- a/web-console/src/druid-models/task/task.ts +++ b/web-console/src/druid-models/task/task.ts @@ -77,6 +77,24 @@ export interface MsqTaskPayloadResponse { }; } +export interface WorkerState { + workerId: string; + state: string; + durationMs: number; +} + +export interface SegmentLoadWaiterStatus { + state: 'INIT' | 'WAITING' | 'SUCCESS'; + startTime: string; + duration: number; + totalSegments: number; + usedSegments: number; + precachedSegments: number; + onDemandSegments: number; + pendingSegments: number; + unknownSegments: number; +} + export interface MsqTaskReportResponse { multiStageQuery: { type: 'multiStageQuery'; @@ -90,6 +108,8 @@ export interface MsqTaskReportResponse { durationMs: number; pendingTasks: number; runningTasks: number; + workers?: Record; + segmentLoadWaiterStatus?: SegmentLoadWaiterStatus; }; stages: StageDefinition[]; counters: Record>; diff --git a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap index df9acdfb2cc..ad8199d56bf 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap @@ -40,7 +40,7 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` >

- General info for query-af8a263d-213f-418e-ad8d-37d55beff59b ingesting into "kttm-blank-lines" + General info for query-26d490c6-c06d-4cd2-938f-bc5f7f982754 ingesting into "kttm-blank-lines"

Results written to dataSource @@ -55,14 +55,14 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` "useLineageBasedSegmentAllocation": true, }, "dataSource": "kttm-blank-lines", - "groupId": "query-af8a263d-213f-418e-ad8d-37d55beff59b", - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "groupId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeTypeNames": Array [ "LONG", "STRING", ], "resource": Object { - "availabilityGroup": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "availabilityGroup": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "requiredCapacity": 1, }, "spec": Object { @@ -91,6 +91,7 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -98,11 +99,12 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -168,15 +170,17 @@ FROM TABLE( ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY DAY", "sqlQueryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "sqlResultsContext": Object { "serializeComplexValues": true, @@ -189,7 +193,7 @@ PARTITIONED BY DAY", ], "type": "query_controller", }, - "task": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "task": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", }, "capacityInfo": undefined, "destination": Object { @@ -201,7 +205,7 @@ PARTITIONED BY DAY", "type": "dataSource", }, "destinationPages": undefined, - "duration": 14545, + "duration": 4483, "engine": "sql-msq-task", "error": Object { "error": Object { @@ -211,15 +215,16 @@ PARTITIONED BY DAY", "rootErrorCode": "CannotParseExternalData", }, "host": "localhost", - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeQuery": Object { "columns": Array [ "agent_type", "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -227,11 +232,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -280,11 +286,13 @@ PARTITIONED BY DAY", ], }, "queryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, + "waitUntilSegmentsLoad": true, }, "result": undefined, "segmentStatus": undefined, @@ -352,7 +360,7 @@ PARTITIONED BY DAY", "stages": Array [ Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_0", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_0", "input": Array [ Object { "inputFormat": Object { @@ -388,6 +396,7 @@ PARTITIONED BY DAY", "v0", ], "context": Object { + "__resultFormat": "array", "__timeColumn": "v0", "__user": "allowAll", "executionMode": "async", @@ -396,11 +405,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -487,16 +497,16 @@ PARTITIONED BY DAY", }, ], }, - "duration": 14193, + "duration": 4259, "phase": "FAILED", "sort": true, "stageNumber": 0, - "startTime": "2023-08-01T04:20:25.296Z", + "startTime": "2024-01-23T19:56:44.857Z", "workerCount": 1, }, Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_1", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_1", "input": Array [ Object { "stage": 0, @@ -566,11 +576,11 @@ PARTITIONED BY DAY", }, ], }, - "startTime": 2023-08-01T04:20:24.945Z, + "startTime": 2024-01-23T19:56:44.633Z, "status": "FAILED", "usageInfo": Object { "pendingTasks": 0, - "runningTasks": 2, + "runningTasks": 1, }, "warnings": Array [ Object { @@ -580,44 +590,45 @@ PARTITIONED BY DAY", }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, Object { "error": Object { @@ -626,44 +637,45 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, ], } @@ -679,14 +691,14 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte "useLineageBasedSegmentAllocation": true, }, "dataSource": "kttm-blank-lines", - "groupId": "query-af8a263d-213f-418e-ad8d-37d55beff59b", - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "groupId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeTypeNames": Array [ "LONG", "STRING", ], "resource": Object { - "availabilityGroup": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "availabilityGroup": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "requiredCapacity": 1, }, "spec": Object { @@ -715,6 +727,7 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -722,11 +735,12 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -792,15 +806,17 @@ FROM TABLE( ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY DAY", "sqlQueryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "sqlResultsContext": Object { "serializeComplexValues": true, @@ -813,7 +829,7 @@ PARTITIONED BY DAY", ], "type": "query_controller", }, - "task": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "task": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", }, "capacityInfo": undefined, "destination": Object { @@ -825,7 +841,7 @@ PARTITIONED BY DAY", "type": "dataSource", }, "destinationPages": undefined, - "duration": 14545, + "duration": 4483, "engine": "sql-msq-task", "error": Object { "error": Object { @@ -835,15 +851,16 @@ PARTITIONED BY DAY", "rootErrorCode": "CannotParseExternalData", }, "host": "localhost", - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeQuery": Object { "columns": Array [ "agent_type", "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -851,11 +868,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -904,11 +922,13 @@ PARTITIONED BY DAY", ], }, "queryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, + "waitUntilSegmentsLoad": true, }, "result": undefined, "segmentStatus": undefined, @@ -976,7 +996,7 @@ PARTITIONED BY DAY", "stages": Array [ Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_0", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_0", "input": Array [ Object { "inputFormat": Object { @@ -1012,6 +1032,7 @@ PARTITIONED BY DAY", "v0", ], "context": Object { + "__resultFormat": "array", "__timeColumn": "v0", "__user": "allowAll", "executionMode": "async", @@ -1020,11 +1041,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -1111,16 +1133,16 @@ PARTITIONED BY DAY", }, ], }, - "duration": 14193, + "duration": 4259, "phase": "FAILED", "sort": true, "stageNumber": 0, - "startTime": "2023-08-01T04:20:25.296Z", + "startTime": "2024-01-23T19:56:44.857Z", "workerCount": 1, }, Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_1", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_1", "input": Array [ Object { "stage": 0, @@ -1190,11 +1212,11 @@ PARTITIONED BY DAY", }, ], }, - "startTime": 2023-08-01T04:20:24.945Z, + "startTime": 2024-01-23T19:56:44.633Z, "status": "FAILED", "usageInfo": Object { "pendingTasks": 0, - "runningTasks": 2, + "runningTasks": 1, }, "warnings": Array [ Object { @@ -1204,44 +1226,45 @@ PARTITIONED BY DAY", }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, Object { "error": Object { @@ -1250,44 +1273,45 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, ], } @@ -1348,14 +1372,14 @@ exports[`ExecutionDetailsPane matches snapshot with init tab 1`] = ` "useLineageBasedSegmentAllocation": true, }, "dataSource": "kttm-blank-lines", - "groupId": "query-af8a263d-213f-418e-ad8d-37d55beff59b", - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "groupId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeTypeNames": Array [ "LONG", "STRING", ], "resource": Object { - "availabilityGroup": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "availabilityGroup": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "requiredCapacity": 1, }, "spec": Object { @@ -1384,6 +1408,7 @@ exports[`ExecutionDetailsPane matches snapshot with init tab 1`] = ` "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -1391,11 +1416,12 @@ exports[`ExecutionDetailsPane matches snapshot with init tab 1`] = ` "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -1461,15 +1487,17 @@ FROM TABLE( ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY DAY", "sqlQueryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "sqlResultsContext": Object { "serializeComplexValues": true, @@ -1482,7 +1510,7 @@ PARTITIONED BY DAY", ], "type": "query_controller", }, - "task": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "task": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", }, "capacityInfo": undefined, "destination": Object { @@ -1494,7 +1522,7 @@ PARTITIONED BY DAY", "type": "dataSource", }, "destinationPages": undefined, - "duration": 14545, + "duration": 4483, "engine": "sql-msq-task", "error": Object { "error": Object { @@ -1504,15 +1532,16 @@ PARTITIONED BY DAY", "rootErrorCode": "CannotParseExternalData", }, "host": "localhost", - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, - "id": "query-af8a263d-213f-418e-ad8d-37d55beff59b", + "id": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754", "nativeQuery": Object { "columns": Array [ "agent_type", "v0", ], "context": Object { + "__resultFormat": "array", "__user": "allowAll", "executionMode": "async", "finalize": false, @@ -1520,11 +1549,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -1573,11 +1603,13 @@ PARTITIONED BY DAY", ], }, "queryContext": Object { + "__resultFormat": "array", "executionMode": "async", "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, + "waitUntilSegmentsLoad": true, }, "result": undefined, "segmentStatus": undefined, @@ -1645,7 +1677,7 @@ PARTITIONED BY DAY", "stages": Array [ Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_0", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_0", "input": Array [ Object { "inputFormat": Object { @@ -1681,6 +1713,7 @@ PARTITIONED BY DAY", "v0", ], "context": Object { + "__resultFormat": "array", "__timeColumn": "v0", "__user": "allowAll", "executionMode": "async", @@ -1689,11 +1722,12 @@ PARTITIONED BY DAY", "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 2, - "queryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "queryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "\\"DAY\\"", - "sqlQueryId": "af8a263d-213f-418e-ad8d-37d55beff59b", + "sqlQueryId": "26d490c6-c06d-4cd2-938f-bc5f7f982754", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -1780,16 +1814,16 @@ PARTITIONED BY DAY", }, ], }, - "duration": 14193, + "duration": 4259, "phase": "FAILED", "sort": true, "stageNumber": 0, - "startTime": "2023-08-01T04:20:25.296Z", + "startTime": "2024-01-23T19:56:44.857Z", "workerCount": 1, }, Object { "definition": Object { - "id": "f635e36d-6b90-4b74-ad5e-a179c99f0ddc_1", + "id": "2c45d26d-8a63-45a7-ae8d-2a2a4d6309b7_1", "input": Array [ Object { "stage": 0, @@ -1859,11 +1893,11 @@ PARTITIONED BY DAY", }, ], }, - "startTime": 2023-08-01T04:20:24.945Z, + "startTime": 2024-01-23T19:56:44.633Z, "status": "FAILED", "usageInfo": Object { "pendingTasks": 0, - "runningTasks": 2, + "runningTasks": 1, }, "warnings": Array [ Object { @@ -1873,44 +1907,45 @@ PARTITIONED BY DAY", }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, Object { "error": Object { @@ -1919,44 +1954,45 @@ Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No conte }, "exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74) - at org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108) - at org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72) + at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93) + at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42) at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:95) at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115) - at org.apache.druid.segment.RowWalker.advance(RowWalker.java:70) + at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75) at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110) at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:275) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:242) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:181) - at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159) - at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:146) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:362) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:329) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:268) + at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:89) + at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:158) at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229) - at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137) - at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:837) - at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230) + at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138) + at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:840) + at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - at java.base/java.lang.Thread.run(Thread.java:829) + at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) + at java.base/java.lang.Thread.run(Thread.java:840) Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input at [Source: (String)\\"\\"; line: 1, column: 0] at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) - at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360) - at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214) - at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182) + at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3548) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3516) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75) at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48) at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71) - ... 23 more + ... 24 more ", "host": "localhost:8101", "stageNumber": 0, - "taskId": "query-af8a263d-213f-418e-ad8d-37d55beff59b-worker0_0", + "taskId": "query-26d490c6-c06d-4cd2-938f-bc5f7f982754-worker0_0", }, ], } diff --git a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap index f6000ab38d4..9d133e240ab 100644 --- a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap @@ -21,7 +21,7 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `

Failed task ID: (on host: diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap index 3e2b45ec2f1..ec8542eb1eb 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap @@ -21,7 +21,7 @@ exports[`ExecutionProgressBarPane matches snapshot 1`] = ` intent="primary" /> - + Segments loaded successfully in 0:00:05
`; diff --git a/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap index a5018aec85c..2225217a428 100644 --- a/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap @@ -158,7 +158,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` Array [ Object { "definition": Object { - "id": "ad318360-2ccf-4afc-b221-27c8704bf4fe_0", + "id": "7f62fa91-f49a-4053-adec-5aa09c251ee3_0", "input": Array [ Object { "inputFormat": Object { @@ -194,6 +194,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` "v0", ], "context": Object { + "__resultFormat": "array", "__timeColumn": "v0", "__user": "allowAll", "executionMode": "async", @@ -202,11 +203,12 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` "groupByEnableMultiValueUnnesting": false, "maxNumTasks": 2, "maxParseExceptions": 0, - "queryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "queryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "93a855fa-c35a-48df-b596-6bc98eed1101", + "sqlQueryId": "346b9ac6-4912-46e4-9b98-75f11071af87", "sqlReplaceTimeChunks": "all", + "waitUntilSegmentsLoad": true, }, "dataSource": Object { "inputFormat": Object { @@ -284,17 +286,17 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` }, ], }, - "duration": 21324, + "duration": 6884, "partitionCount": 1, "phase": "FINISHED", "sort": true, "stageNumber": 0, - "startTime": "2023-08-01T03:12:59.865Z", + "startTime": "2024-01-23T19:45:43.302Z", "workerCount": 1, }, Object { "definition": Object { - "id": "ad318360-2ccf-4afc-b221-27c8704bf4fe_1", + "id": "7f62fa91-f49a-4053-adec-5aa09c251ee3_1", "input": Array [ Object { "stage": 0, @@ -360,11 +362,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` }, "signature": Array [], }, - "duration": 2070, + "duration": 1263, "partitionCount": 1, "phase": "FINISHED", "stageNumber": 1, - "startTime": "2023-08-01T03:13:21.156Z", + "startTime": "2024-01-23T19:45:50.170Z", "workerCount": 1, }, ] diff --git a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx index 1c06446eb13..88cefa045ad 100644 --- a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx +++ b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx @@ -20,6 +20,7 @@ import { Button, Icon, Intent } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { Tooltip2 } from '@blueprintjs/popover2'; import classNames from 'classnames'; +import * as JSONBig from 'json-bigint-native'; import React from 'react'; import type { Column } from 'react-table'; import ReactTable from 'react-table'; @@ -34,11 +35,13 @@ import type { SegmentGenerationProgressFields, SimpleWideCounter, StageDefinition, + StageInput, } from '../../../druid-models'; import { formatClusterBy, Stages, summarizeInputSource } from '../../../druid-models'; import { DEFAULT_TABLE_CLASS_NAME } from '../../../react-table'; import type { NumberLike } from '../../../utils'; import { + assemble, capitalizeFirst, clamp, deepGet, @@ -59,6 +62,15 @@ const MAX_STAGE_ROWS = 20; const MAX_DETAIL_ROWS = 20; const NOT_SIZE_ON_DISK = '(does not represent size on disk)'; +function summarizeTableInput(tableStageInput: StageInput): string { + if (tableStageInput.type !== 'table') return ''; + return assemble( + `Datasource: ${tableStageInput.dataSource}`, + `Interval: ${tableStageInput.intervals.join('; ')}`, + tableStageInput.filter && `Filter: ${JSONBig.stringify(tableStageInput.filter)}`, + ).join('\n'); +} + function formatBreakdown(breakdown: Record): string { return Object.keys(breakdown) .map(k => `${k}: ${formatInteger(breakdown[k])}`) @@ -85,7 +97,7 @@ function inputLabelContent(stage: StageDefinition, inputIndex: number) { Input{' '} {stageInput.type === 'stage' && {`Stage${stageInput.stage}`}} {stageInput.type === 'table' && ( - + {stageInput.dataSource} )} diff --git a/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap b/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap index 7c6d702e560..f8aea105f41 100644 --- a/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap @@ -8,8 +8,8 @@ exports[`IngestSuccessPane matches snapshot 1`] = ` 465,346 rows inserted into "kttm_simple".

- Insert query took 0:00:23. - + Insert query took 0:00:14. + Segments loaded successfully in 0:00:05