mirror of https://github.com/apache/druid.git
Web console: use stages, counters, and warnings from the new detailed status API (#16809)
* stages and counters can be seen on the status reponse * warnings are exposed also * mark as msq when attached * update snapshots * download CSV/TSV null as empty cell
This commit is contained in:
Vadim Ogievetsky
GitHub
parent
bb4d6cc001
commit
8c170f7d0e
|
|
@ -29,9 +29,9 @@ LIMIT 2
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = {
|
export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = {
|
||||||
queryId: 'query-ad84d20a-c331-4ee9-ac59-83024e369cf1',
|
queryId: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349',
|
||||||
state: 'SUCCESS',
|
state: 'SUCCESS',
|
||||||
createdAt: '2023-07-05T21:33:19.147Z',
|
createdAt: '2024-07-27T02:39:22.230Z',
|
||||||
schema: [
|
schema: [
|
||||||
{
|
{
|
||||||
name: 'channel',
|
name: 'channel',
|
||||||
|
|
@ -44,10 +44,10 @@ export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = {
|
||||||
nativeType: 'LONG',
|
nativeType: 'LONG',
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
durationMs: 29168,
|
durationMs: 7183,
|
||||||
result: {
|
result: {
|
||||||
numTotalRows: 2,
|
numTotalRows: 2,
|
||||||
totalSizeInBytes: 116,
|
totalSizeInBytes: 150,
|
||||||
dataSource: '__query_select',
|
dataSource: '__query_select',
|
||||||
sampleRecords: [
|
sampleRecords: [
|
||||||
['#en.wikipedia', 6650],
|
['#en.wikipedia', 6650],
|
||||||
|
|
@ -55,12 +55,395 @@ export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = {
|
||||||
],
|
],
|
||||||
pages: [
|
pages: [
|
||||||
{
|
{
|
||||||
numRows: 2,
|
|
||||||
sizeInBytes: 116,
|
|
||||||
id: 0,
|
id: 0,
|
||||||
|
numRows: 2,
|
||||||
|
sizeInBytes: 150,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
stages: [
|
||||||
|
{
|
||||||
|
stageNumber: 0,
|
||||||
|
definition: {
|
||||||
|
id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_0',
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: 'table',
|
||||||
|
dataSource: 'wikipedia',
|
||||||
|
intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
processor: {
|
||||||
|
type: 'groupByPreShuffle',
|
||||||
|
query: {
|
||||||
|
queryType: 'groupBy',
|
||||||
|
dataSource: {
|
||||||
|
type: 'inputNumber',
|
||||||
|
inputNumber: 0,
|
||||||
|
},
|
||||||
|
intervals: {
|
||||||
|
type: 'intervals',
|
||||||
|
intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
|
||||||
|
},
|
||||||
|
granularity: {
|
||||||
|
type: 'all',
|
||||||
|
},
|
||||||
|
dimensions: [
|
||||||
|
{
|
||||||
|
type: 'default',
|
||||||
|
dimension: 'channel',
|
||||||
|
outputName: 'd0',
|
||||||
|
outputType: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
aggregations: [
|
||||||
|
{
|
||||||
|
type: 'count',
|
||||||
|
name: 'a0',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
limitSpec: {
|
||||||
|
type: 'default',
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
dimension: 'a0',
|
||||||
|
direction: 'descending',
|
||||||
|
dimensionOrder: {
|
||||||
|
type: 'numeric',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
context: {
|
||||||
|
__resultFormat: 'array',
|
||||||
|
__user: 'allowAll',
|
||||||
|
executionMode: 'async',
|
||||||
|
finalize: true,
|
||||||
|
maxNumTasks: 2,
|
||||||
|
maxParseExceptions: 0,
|
||||||
|
queryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349',
|
||||||
|
sqlOuterLimit: 1001,
|
||||||
|
sqlQueryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349',
|
||||||
|
sqlStringifyArrays: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: 'd0',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'a0',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
shuffleSpec: {
|
||||||
|
type: 'maxCount',
|
||||||
|
clusterBy: {
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
columnName: 'd0',
|
||||||
|
order: 'ASCENDING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
partitions: 1,
|
||||||
|
aggregate: true,
|
||||||
|
},
|
||||||
|
maxWorkerCount: 1,
|
||||||
|
},
|
||||||
|
phase: 'FINISHED',
|
||||||
|
workerCount: 1,
|
||||||
|
partitionCount: 1,
|
||||||
|
shuffle: 'globalSort',
|
||||||
|
output: 'localStorage',
|
||||||
|
startTime: '2024-07-27T02:39:24.713Z',
|
||||||
|
duration: 3384,
|
||||||
|
sort: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
stageNumber: 1,
|
||||||
|
definition: {
|
||||||
|
id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_1',
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: 'stage',
|
||||||
|
stage: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
processor: {
|
||||||
|
type: 'groupByPostShuffle',
|
||||||
|
query: {
|
||||||
|
queryType: 'groupBy',
|
||||||
|
dataSource: {
|
||||||
|
type: 'inputNumber',
|
||||||
|
inputNumber: 0,
|
||||||
|
},
|
||||||
|
intervals: {
|
||||||
|
type: 'intervals',
|
||||||
|
intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
|
||||||
|
},
|
||||||
|
granularity: {
|
||||||
|
type: 'all',
|
||||||
|
},
|
||||||
|
dimensions: [
|
||||||
|
{
|
||||||
|
type: 'default',
|
||||||
|
dimension: 'channel',
|
||||||
|
outputName: 'd0',
|
||||||
|
outputType: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
aggregations: [
|
||||||
|
{
|
||||||
|
type: 'count',
|
||||||
|
name: 'a0',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
limitSpec: {
|
||||||
|
type: 'default',
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
dimension: 'a0',
|
||||||
|
direction: 'descending',
|
||||||
|
dimensionOrder: {
|
||||||
|
type: 'numeric',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
context: {
|
||||||
|
__resultFormat: 'array',
|
||||||
|
__user: 'allowAll',
|
||||||
|
executionMode: 'async',
|
||||||
|
finalize: true,
|
||||||
|
maxNumTasks: 2,
|
||||||
|
maxParseExceptions: 0,
|
||||||
|
queryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349',
|
||||||
|
sqlOuterLimit: 1001,
|
||||||
|
sqlQueryId: '45f1dafd-8a52-4eb7-9a6c-77840cddd349',
|
||||||
|
sqlStringifyArrays: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: 'a0',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '__boost',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'd0',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
shuffleSpec: {
|
||||||
|
type: 'maxCount',
|
||||||
|
clusterBy: {
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
columnName: 'a0',
|
||||||
|
order: 'DESCENDING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
columnName: '__boost',
|
||||||
|
order: 'ASCENDING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
partitions: 1,
|
||||||
|
},
|
||||||
|
maxWorkerCount: 1,
|
||||||
|
},
|
||||||
|
phase: 'FINISHED',
|
||||||
|
workerCount: 1,
|
||||||
|
partitionCount: 1,
|
||||||
|
shuffle: 'globalSort',
|
||||||
|
output: 'localStorage',
|
||||||
|
startTime: '2024-07-27T02:39:28.089Z',
|
||||||
|
duration: 26,
|
||||||
|
sort: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
stageNumber: 2,
|
||||||
|
definition: {
|
||||||
|
id: 'query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_2',
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: 'stage',
|
||||||
|
stage: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
processor: {
|
||||||
|
type: 'limit',
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: 'a0',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '__boost',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'd0',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
shuffleSpec: {
|
||||||
|
type: 'maxCount',
|
||||||
|
clusterBy: {
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
columnName: 'a0',
|
||||||
|
order: 'DESCENDING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
columnName: '__boost',
|
||||||
|
order: 'ASCENDING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
partitions: 1,
|
||||||
|
},
|
||||||
|
maxWorkerCount: 1,
|
||||||
|
},
|
||||||
|
phase: 'FINISHED',
|
||||||
|
workerCount: 1,
|
||||||
|
partitionCount: 1,
|
||||||
|
shuffle: 'globalSort',
|
||||||
|
output: 'localStorage',
|
||||||
|
startTime: '2024-07-27T02:39:28.112Z',
|
||||||
|
duration: 12,
|
||||||
|
sort: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
counters: {
|
||||||
|
'0': {
|
||||||
|
'0': {
|
||||||
|
input0: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [24433],
|
||||||
|
bytes: [6525055],
|
||||||
|
files: [1],
|
||||||
|
totalFiles: [1],
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2335],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
shuffle: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2131],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
sortProgress: {
|
||||||
|
type: 'sortProgress',
|
||||||
|
totalMergingLevels: 3,
|
||||||
|
levelToTotalBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
levelToMergedBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
totalMergersForUltimateLevel: 1,
|
||||||
|
progressDigest: 1.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'1': {
|
||||||
|
'0': {
|
||||||
|
input0: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2131],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2998],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
shuffle: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2794],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
sortProgress: {
|
||||||
|
type: 'sortProgress',
|
||||||
|
totalMergingLevels: 3,
|
||||||
|
levelToTotalBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
levelToMergedBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
totalMergersForUltimateLevel: 1,
|
||||||
|
progressDigest: 1.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'2': {
|
||||||
|
'0': {
|
||||||
|
input0: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [51],
|
||||||
|
bytes: [2794],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [2],
|
||||||
|
bytes: [150],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
shuffle: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [2],
|
||||||
|
bytes: [142],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
sortProgress: {
|
||||||
|
type: 'sortProgress',
|
||||||
|
totalMergingLevels: 3,
|
||||||
|
levelToTotalBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
levelToMergedBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': 1,
|
||||||
|
},
|
||||||
|
totalMergersForUltimateLevel: 1,
|
||||||
|
progressDigest: 1.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
@ -82,18 +465,285 @@ PARTITIONED BY DAY
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const FAILED_ASYNC_STATUS: AsyncStatusResponse = {
|
export const FAILED_ASYNC_STATUS: AsyncStatusResponse = {
|
||||||
queryId: 'query-36ea273a-bd6d-48de-b890-2d853d879bf8',
|
queryId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4',
|
||||||
state: 'FAILED',
|
state: 'FAILED',
|
||||||
createdAt: '2023-07-05T21:40:39.986Z',
|
createdAt: '2024-07-26T18:04:59.873Z',
|
||||||
durationMs: 11217,
|
durationMs: 6954,
|
||||||
errorDetails: {
|
errorDetails: {
|
||||||
error: 'druidException',
|
error: 'druidException',
|
||||||
errorCode: 'UnknownError',
|
errorCode: 'TooManyWarnings',
|
||||||
persona: 'USER',
|
persona: 'USER',
|
||||||
category: 'UNCATEGORIZED',
|
category: 'UNCATEGORIZED',
|
||||||
errorMessage: 'java.io.UncheckedIOException: /',
|
errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)',
|
||||||
context: {
|
context: {
|
||||||
message: 'java.io.UncheckedIOException: /',
|
maxWarnings: '2',
|
||||||
|
rootErrorCode: 'CannotParseExternalData',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
stages: [
|
||||||
|
{
|
||||||
|
stageNumber: 0,
|
||||||
|
definition: {
|
||||||
|
id: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4_0',
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: 'external',
|
||||||
|
inputSource: {
|
||||||
|
type: 'http',
|
||||||
|
uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'],
|
||||||
|
},
|
||||||
|
inputFormat: {
|
||||||
|
type: 'json',
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: 'timestamp',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'agent_type',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
processor: {
|
||||||
|
type: 'scan',
|
||||||
|
query: {
|
||||||
|
queryType: 'scan',
|
||||||
|
dataSource: {
|
||||||
|
type: 'external',
|
||||||
|
inputSource: {
|
||||||
|
type: 'http',
|
||||||
|
uris: [
|
||||||
|
'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
inputFormat: {
|
||||||
|
type: 'json',
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: 'timestamp',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'agent_type',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
intervals: {
|
||||||
|
type: 'intervals',
|
||||||
|
intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
|
||||||
|
},
|
||||||
|
virtualColumns: [
|
||||||
|
{
|
||||||
|
type: 'expression',
|
||||||
|
name: 'v0',
|
||||||
|
expression: 'timestamp_parse("timestamp",null,\'UTC\')',
|
||||||
|
outputType: 'LONG',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
resultFormat: 'compactedList',
|
||||||
|
columns: ['agent_type', 'v0'],
|
||||||
|
context: {
|
||||||
|
__resultFormat: 'array',
|
||||||
|
__timeColumn: 'v0',
|
||||||
|
__user: 'allowAll',
|
||||||
|
executionMode: 'async',
|
||||||
|
finalize: false,
|
||||||
|
finalizeAggregations: false,
|
||||||
|
groupByEnableMultiValueUnnesting: false,
|
||||||
|
maxNumTasks: 2,
|
||||||
|
maxParseExceptions: 2,
|
||||||
|
queryId: 'ea3e36df-ad67-4870-b136-f5616b17d9c4',
|
||||||
|
scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
|
||||||
|
sqlInsertSegmentGranularity: '"DAY"',
|
||||||
|
sqlQueryId: 'ea3e36df-ad67-4870-b136-f5616b17d9c4',
|
||||||
|
sqlReplaceTimeChunks: 'all',
|
||||||
|
sqlStringifyArrays: false,
|
||||||
|
waitUntilSegmentsLoad: true,
|
||||||
|
},
|
||||||
|
columnTypes: ['STRING', 'LONG'],
|
||||||
|
granularity: {
|
||||||
|
type: 'all',
|
||||||
|
},
|
||||||
|
legacy: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
signature: [
|
||||||
|
{
|
||||||
|
name: '__bucket',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: '__boost',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'agent_type',
|
||||||
|
type: 'STRING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'v0',
|
||||||
|
type: 'LONG',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
shuffleSpec: {
|
||||||
|
type: 'targetSize',
|
||||||
|
clusterBy: {
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
columnName: '__bucket',
|
||||||
|
order: 'ASCENDING',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
columnName: '__boost',
|
||||||
|
order: 'ASCENDING',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
bucketByCount: 1,
|
||||||
|
},
|
||||||
|
targetSize: 3000000,
|
||||||
|
},
|
||||||
|
maxWorkerCount: 1,
|
||||||
|
shuffleCheckHasMultipleValues: true,
|
||||||
|
},
|
||||||
|
phase: 'FAILED',
|
||||||
|
workerCount: 1,
|
||||||
|
shuffle: 'globalSort',
|
||||||
|
output: 'localStorage',
|
||||||
|
startTime: '2024-07-26T18:05:02.399Z',
|
||||||
|
duration: 4056,
|
||||||
|
sort: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
stageNumber: 1,
|
||||||
|
definition: {
|
||||||
|
id: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4_1',
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: 'stage',
|
||||||
|
stage: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
processor: {
|
||||||
|
type: 'segmentGenerator',
|
||||||
|
dataSchema: {
|
||||||
|
dataSource: 'kttm-blank-lines',
|
||||||
|
timestampSpec: {
|
||||||
|
column: '__time',
|
||||||
|
format: 'millis',
|
||||||
|
missingValue: null,
|
||||||
|
},
|
||||||
|
dimensionsSpec: {
|
||||||
|
dimensions: [
|
||||||
|
{
|
||||||
|
type: 'string',
|
||||||
|
name: 'agent_type',
|
||||||
|
multiValueHandling: 'SORTED_ARRAY',
|
||||||
|
createBitmapIndex: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
dimensionExclusions: ['__time'],
|
||||||
|
includeAllDimensions: false,
|
||||||
|
useSchemaDiscovery: false,
|
||||||
|
},
|
||||||
|
metricsSpec: [],
|
||||||
|
granularitySpec: {
|
||||||
|
type: 'arbitrary',
|
||||||
|
queryGranularity: {
|
||||||
|
type: 'none',
|
||||||
|
},
|
||||||
|
rollup: false,
|
||||||
|
intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
|
||||||
|
},
|
||||||
|
transformSpec: {
|
||||||
|
filter: null,
|
||||||
|
transforms: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
columnMappings: [
|
||||||
|
{
|
||||||
|
queryColumn: 'v0',
|
||||||
|
outputColumn: '__time',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
queryColumn: 'agent_type',
|
||||||
|
outputColumn: 'agent_type',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
tuningConfig: {
|
||||||
|
maxNumWorkers: 1,
|
||||||
|
maxRowsInMemory: 100000,
|
||||||
|
rowsPerSegment: 3000000,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
signature: [],
|
||||||
|
maxWorkerCount: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
counters: {
|
||||||
|
'0': {
|
||||||
|
'0': {
|
||||||
|
input0: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [10],
|
||||||
|
bytes: [7658],
|
||||||
|
files: [1],
|
||||||
|
totalFiles: [1],
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
type: 'channel',
|
||||||
|
rows: [10],
|
||||||
|
bytes: [712],
|
||||||
|
frames: [1],
|
||||||
|
},
|
||||||
|
sortProgress: {
|
||||||
|
type: 'sortProgress',
|
||||||
|
totalMergingLevels: 3,
|
||||||
|
levelToTotalBatches: {
|
||||||
|
'0': 1,
|
||||||
|
'1': 1,
|
||||||
|
'2': -1,
|
||||||
|
},
|
||||||
|
levelToMergedBatches: {},
|
||||||
|
totalMergersForUltimateLevel: -1,
|
||||||
|
progressDigest: 0.0,
|
||||||
|
},
|
||||||
|
warnings: {
|
||||||
|
type: 'warnings',
|
||||||
|
CannotParseExternalData: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
warnings: [
|
||||||
|
{
|
||||||
|
taskId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0',
|
||||||
|
host: 'localhost:8101',
|
||||||
|
stageNumber: 0,
|
||||||
|
error: {
|
||||||
|
errorCode: 'CannotParseExternalData',
|
||||||
|
errorMessage:
|
||||||
|
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)',
|
||||||
|
},
|
||||||
|
exceptionStackTrace:
|
||||||
|
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (byte[])""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
taskId: 'query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0',
|
||||||
|
host: 'localhost:8101',
|
||||||
|
stageNumber: 0,
|
||||||
|
error: {
|
||||||
|
errorCode: 'CannotParseExternalData',
|
||||||
|
errorMessage:
|
||||||
|
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)',
|
||||||
|
},
|
||||||
|
exceptionStackTrace:
|
||||||
|
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)\n\tat org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (byte[])""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 24 more\n',
|
||||||
|
},
|
||||||
|
],
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { ErrorResponse } from '../../utils';
|
import type { ErrorResponse } from '../../utils';
|
||||||
|
import type { Counters, StageDefinition } from '../stages/stages';
|
||||||
|
import type { MsqTaskErrorReport } from '../task/task';
|
||||||
|
|
||||||
export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'SUCCESS' | 'FAILED';
|
export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'SUCCESS' | 'FAILED';
|
||||||
|
|
||||||
|
|
@ -37,5 +39,8 @@ export interface AsyncStatusResponse {
|
||||||
sizeInBytes: number;
|
sizeInBytes: number;
|
||||||
}[];
|
}[];
|
||||||
};
|
};
|
||||||
|
stages?: StageDefinition[];
|
||||||
|
counters?: Counters;
|
||||||
errorDetails?: ErrorResponse;
|
errorDetails?: ErrorResponse;
|
||||||
|
warnings?: MsqTaskErrorReport[];
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -622,13 +622,13 @@ describe('Execution', () => {
|
||||||
{
|
{
|
||||||
"id": 0,
|
"id": 0,
|
||||||
"numRows": 2,
|
"numRows": 2,
|
||||||
"sizeInBytes": 116,
|
"sizeInBytes": 150,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"duration": 29168,
|
"duration": 7183,
|
||||||
"engine": "sql-msq-task",
|
"engine": "sql-msq-task",
|
||||||
"error": undefined,
|
"error": undefined,
|
||||||
"id": "query-ad84d20a-c331-4ee9-ac59-83024e369cf1",
|
"id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349",
|
||||||
"nativeQuery": undefined,
|
"nativeQuery": undefined,
|
||||||
"queryContext": undefined,
|
"queryContext": undefined,
|
||||||
"result": _QueryResult {
|
"result": _QueryResult {
|
||||||
|
|
@ -663,8 +663,453 @@ describe('Execution', () => {
|
||||||
},
|
},
|
||||||
"segmentStatus": undefined,
|
"segmentStatus": undefined,
|
||||||
"sqlQuery": undefined,
|
"sqlQuery": undefined,
|
||||||
"stages": undefined,
|
"stages": Stages {
|
||||||
"startTime": 2023-07-05T21:33:19.147Z,
|
"counters": {
|
||||||
|
"0": {
|
||||||
|
"0": {
|
||||||
|
"input0": {
|
||||||
|
"bytes": [
|
||||||
|
6525055,
|
||||||
|
],
|
||||||
|
"files": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
24433,
|
||||||
|
],
|
||||||
|
"totalFiles": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"bytes": [
|
||||||
|
2335,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"shuffle": {
|
||||||
|
"bytes": [
|
||||||
|
2131,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"sortProgress": {
|
||||||
|
"levelToMergedBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"levelToTotalBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"progressDigest": 1,
|
||||||
|
"totalMergersForUltimateLevel": 1,
|
||||||
|
"totalMergingLevels": 3,
|
||||||
|
"type": "sortProgress",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"0": {
|
||||||
|
"input0": {
|
||||||
|
"bytes": [
|
||||||
|
2131,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"bytes": [
|
||||||
|
2998,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"shuffle": {
|
||||||
|
"bytes": [
|
||||||
|
2794,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"sortProgress": {
|
||||||
|
"levelToMergedBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"levelToTotalBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"progressDigest": 1,
|
||||||
|
"totalMergersForUltimateLevel": 1,
|
||||||
|
"totalMergingLevels": 3,
|
||||||
|
"type": "sortProgress",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"0": {
|
||||||
|
"input0": {
|
||||||
|
"bytes": [
|
||||||
|
2794,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
51,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"bytes": [
|
||||||
|
150,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
2,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"shuffle": {
|
||||||
|
"bytes": [
|
||||||
|
142,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
2,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"sortProgress": {
|
||||||
|
"levelToMergedBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"levelToTotalBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": 1,
|
||||||
|
},
|
||||||
|
"progressDigest": 1,
|
||||||
|
"totalMergersForUltimateLevel": 1,
|
||||||
|
"totalMergingLevels": 3,
|
||||||
|
"type": "sortProgress",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"stages": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_0",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"dataSource": "wikipedia",
|
||||||
|
"intervals": [
|
||||||
|
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z",
|
||||||
|
],
|
||||||
|
"type": "table",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"maxWorkerCount": 1,
|
||||||
|
"processor": {
|
||||||
|
"query": {
|
||||||
|
"aggregations": [
|
||||||
|
{
|
||||||
|
"name": "a0",
|
||||||
|
"type": "count",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"context": {
|
||||||
|
"__resultFormat": "array",
|
||||||
|
"__user": "allowAll",
|
||||||
|
"executionMode": "async",
|
||||||
|
"finalize": true,
|
||||||
|
"maxNumTasks": 2,
|
||||||
|
"maxParseExceptions": 0,
|
||||||
|
"queryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349",
|
||||||
|
"sqlOuterLimit": 1001,
|
||||||
|
"sqlQueryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349",
|
||||||
|
"sqlStringifyArrays": false,
|
||||||
|
},
|
||||||
|
"dataSource": {
|
||||||
|
"inputNumber": 0,
|
||||||
|
"type": "inputNumber",
|
||||||
|
},
|
||||||
|
"dimensions": [
|
||||||
|
{
|
||||||
|
"dimension": "channel",
|
||||||
|
"outputName": "d0",
|
||||||
|
"outputType": "STRING",
|
||||||
|
"type": "default",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"granularity": {
|
||||||
|
"type": "all",
|
||||||
|
},
|
||||||
|
"intervals": {
|
||||||
|
"intervals": [
|
||||||
|
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z",
|
||||||
|
],
|
||||||
|
"type": "intervals",
|
||||||
|
},
|
||||||
|
"limitSpec": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"dimension": "a0",
|
||||||
|
"dimensionOrder": {
|
||||||
|
"type": "numeric",
|
||||||
|
},
|
||||||
|
"direction": "descending",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"limit": 2,
|
||||||
|
"type": "default",
|
||||||
|
},
|
||||||
|
"queryType": "groupBy",
|
||||||
|
},
|
||||||
|
"type": "groupByPreShuffle",
|
||||||
|
},
|
||||||
|
"shuffleSpec": {
|
||||||
|
"aggregate": true,
|
||||||
|
"clusterBy": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"columnName": "d0",
|
||||||
|
"order": "ASCENDING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"partitions": 1,
|
||||||
|
"type": "maxCount",
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "d0",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "a0",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"duration": 3384,
|
||||||
|
"output": "localStorage",
|
||||||
|
"partitionCount": 1,
|
||||||
|
"phase": "FINISHED",
|
||||||
|
"shuffle": "globalSort",
|
||||||
|
"sort": true,
|
||||||
|
"stageNumber": 0,
|
||||||
|
"startTime": "2024-07-27T02:39:24.713Z",
|
||||||
|
"workerCount": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_1",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"stage": 0,
|
||||||
|
"type": "stage",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"maxWorkerCount": 1,
|
||||||
|
"processor": {
|
||||||
|
"query": {
|
||||||
|
"aggregations": [
|
||||||
|
{
|
||||||
|
"name": "a0",
|
||||||
|
"type": "count",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"context": {
|
||||||
|
"__resultFormat": "array",
|
||||||
|
"__user": "allowAll",
|
||||||
|
"executionMode": "async",
|
||||||
|
"finalize": true,
|
||||||
|
"maxNumTasks": 2,
|
||||||
|
"maxParseExceptions": 0,
|
||||||
|
"queryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349",
|
||||||
|
"sqlOuterLimit": 1001,
|
||||||
|
"sqlQueryId": "45f1dafd-8a52-4eb7-9a6c-77840cddd349",
|
||||||
|
"sqlStringifyArrays": false,
|
||||||
|
},
|
||||||
|
"dataSource": {
|
||||||
|
"inputNumber": 0,
|
||||||
|
"type": "inputNumber",
|
||||||
|
},
|
||||||
|
"dimensions": [
|
||||||
|
{
|
||||||
|
"dimension": "channel",
|
||||||
|
"outputName": "d0",
|
||||||
|
"outputType": "STRING",
|
||||||
|
"type": "default",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"granularity": {
|
||||||
|
"type": "all",
|
||||||
|
},
|
||||||
|
"intervals": {
|
||||||
|
"intervals": [
|
||||||
|
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z",
|
||||||
|
],
|
||||||
|
"type": "intervals",
|
||||||
|
},
|
||||||
|
"limitSpec": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"dimension": "a0",
|
||||||
|
"dimensionOrder": {
|
||||||
|
"type": "numeric",
|
||||||
|
},
|
||||||
|
"direction": "descending",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"limit": 2,
|
||||||
|
"type": "default",
|
||||||
|
},
|
||||||
|
"queryType": "groupBy",
|
||||||
|
},
|
||||||
|
"type": "groupByPostShuffle",
|
||||||
|
},
|
||||||
|
"shuffleSpec": {
|
||||||
|
"clusterBy": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"columnName": "a0",
|
||||||
|
"order": "DESCENDING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"columnName": "__boost",
|
||||||
|
"order": "ASCENDING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"partitions": 1,
|
||||||
|
"type": "maxCount",
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "a0",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "__boost",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "d0",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"duration": 26,
|
||||||
|
"output": "localStorage",
|
||||||
|
"partitionCount": 1,
|
||||||
|
"phase": "FINISHED",
|
||||||
|
"shuffle": "globalSort",
|
||||||
|
"sort": true,
|
||||||
|
"stageNumber": 1,
|
||||||
|
"startTime": "2024-07-27T02:39:28.089Z",
|
||||||
|
"workerCount": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"id": "query-45f1dafd-8a52-4eb7-9a6c-77840cddd349_2",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"stage": 1,
|
||||||
|
"type": "stage",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"maxWorkerCount": 1,
|
||||||
|
"processor": {
|
||||||
|
"limit": 2,
|
||||||
|
"type": "limit",
|
||||||
|
},
|
||||||
|
"shuffleSpec": {
|
||||||
|
"clusterBy": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"columnName": "a0",
|
||||||
|
"order": "DESCENDING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"columnName": "__boost",
|
||||||
|
"order": "ASCENDING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"partitions": 1,
|
||||||
|
"type": "maxCount",
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "a0",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "__boost",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "d0",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"duration": 12,
|
||||||
|
"output": "localStorage",
|
||||||
|
"partitionCount": 1,
|
||||||
|
"phase": "FINISHED",
|
||||||
|
"shuffle": "globalSort",
|
||||||
|
"sort": true,
|
||||||
|
"stageNumber": 2,
|
||||||
|
"startTime": "2024-07-27T02:39:28.112Z",
|
||||||
|
"workerCount": 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"startTime": 2024-07-27T02:39:22.230Z,
|
||||||
"status": "SUCCESS",
|
"status": "SUCCESS",
|
||||||
"usageInfo": undefined,
|
"usageInfo": undefined,
|
||||||
"warnings": undefined,
|
"warnings": undefined,
|
||||||
|
|
@ -679,32 +1124,395 @@ describe('Execution', () => {
|
||||||
"capacityInfo": undefined,
|
"capacityInfo": undefined,
|
||||||
"destination": undefined,
|
"destination": undefined,
|
||||||
"destinationPages": undefined,
|
"destinationPages": undefined,
|
||||||
"duration": 11217,
|
"duration": 6954,
|
||||||
"engine": "sql-msq-task",
|
"engine": "sql-msq-task",
|
||||||
"error": {
|
"error": {
|
||||||
"error": {
|
"error": {
|
||||||
"category": "UNCATEGORIZED",
|
"category": "UNCATEGORIZED",
|
||||||
"context": {
|
"context": {
|
||||||
"message": "java.io.UncheckedIOException: /",
|
"maxWarnings": "2",
|
||||||
|
"rootErrorCode": "CannotParseExternalData",
|
||||||
},
|
},
|
||||||
"error": "druidException",
|
"error": "druidException",
|
||||||
"errorCode": "UnknownError",
|
"errorCode": "TooManyWarnings",
|
||||||
"errorMessage": "java.io.UncheckedIOException: /",
|
"errorMessage": "Too many warnings of type CannotParseExternalData generated (max = 2)",
|
||||||
"persona": "USER",
|
"persona": "USER",
|
||||||
},
|
},
|
||||||
"taskId": "query-36ea273a-bd6d-48de-b890-2d853d879bf8",
|
"taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4",
|
||||||
},
|
},
|
||||||
"id": "query-36ea273a-bd6d-48de-b890-2d853d879bf8",
|
"id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4",
|
||||||
"nativeQuery": undefined,
|
"nativeQuery": undefined,
|
||||||
"queryContext": undefined,
|
"queryContext": undefined,
|
||||||
"result": undefined,
|
"result": undefined,
|
||||||
"segmentStatus": undefined,
|
"segmentStatus": undefined,
|
||||||
"sqlQuery": undefined,
|
"sqlQuery": undefined,
|
||||||
"stages": undefined,
|
"stages": Stages {
|
||||||
"startTime": 2023-07-05T21:40:39.986Z,
|
"counters": {
|
||||||
|
"0": {
|
||||||
|
"0": {
|
||||||
|
"input0": {
|
||||||
|
"bytes": [
|
||||||
|
7658,
|
||||||
|
],
|
||||||
|
"files": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
10,
|
||||||
|
],
|
||||||
|
"totalFiles": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"bytes": [
|
||||||
|
712,
|
||||||
|
],
|
||||||
|
"frames": [
|
||||||
|
1,
|
||||||
|
],
|
||||||
|
"rows": [
|
||||||
|
10,
|
||||||
|
],
|
||||||
|
"type": "channel",
|
||||||
|
},
|
||||||
|
"sortProgress": {
|
||||||
|
"levelToMergedBatches": {},
|
||||||
|
"levelToTotalBatches": {
|
||||||
|
"0": 1,
|
||||||
|
"1": 1,
|
||||||
|
"2": -1,
|
||||||
|
},
|
||||||
|
"progressDigest": 0,
|
||||||
|
"totalMergersForUltimateLevel": -1,
|
||||||
|
"totalMergingLevels": 3,
|
||||||
|
"type": "sortProgress",
|
||||||
|
},
|
||||||
|
"warnings": {
|
||||||
|
"CannotParseExternalData": 3,
|
||||||
|
"type": "warnings",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"stages": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4_0",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"inputFormat": {
|
||||||
|
"type": "json",
|
||||||
|
},
|
||||||
|
"inputSource": {
|
||||||
|
"type": "http",
|
||||||
|
"uris": [
|
||||||
|
"https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "timestamp",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "agent_type",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"type": "external",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"maxWorkerCount": 1,
|
||||||
|
"processor": {
|
||||||
|
"query": {
|
||||||
|
"columnTypes": [
|
||||||
|
"STRING",
|
||||||
|
"LONG",
|
||||||
|
],
|
||||||
|
"columns": [
|
||||||
|
"agent_type",
|
||||||
|
"v0",
|
||||||
|
],
|
||||||
|
"context": {
|
||||||
|
"__resultFormat": "array",
|
||||||
|
"__timeColumn": "v0",
|
||||||
|
"__user": "allowAll",
|
||||||
|
"executionMode": "async",
|
||||||
|
"finalize": false,
|
||||||
|
"finalizeAggregations": false,
|
||||||
|
"groupByEnableMultiValueUnnesting": false,
|
||||||
|
"maxNumTasks": 2,
|
||||||
|
"maxParseExceptions": 2,
|
||||||
|
"queryId": "ea3e36df-ad67-4870-b136-f5616b17d9c4",
|
||||||
|
"scanSignature": "[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]",
|
||||||
|
"sqlInsertSegmentGranularity": ""DAY"",
|
||||||
|
"sqlQueryId": "ea3e36df-ad67-4870-b136-f5616b17d9c4",
|
||||||
|
"sqlReplaceTimeChunks": "all",
|
||||||
|
"sqlStringifyArrays": false,
|
||||||
|
"waitUntilSegmentsLoad": true,
|
||||||
|
},
|
||||||
|
"dataSource": {
|
||||||
|
"inputFormat": {
|
||||||
|
"type": "json",
|
||||||
|
},
|
||||||
|
"inputSource": {
|
||||||
|
"type": "http",
|
||||||
|
"uris": [
|
||||||
|
"https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "timestamp",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "agent_type",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"type": "external",
|
||||||
|
},
|
||||||
|
"granularity": {
|
||||||
|
"type": "all",
|
||||||
|
},
|
||||||
|
"intervals": {
|
||||||
|
"intervals": [
|
||||||
|
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z",
|
||||||
|
],
|
||||||
|
"type": "intervals",
|
||||||
|
},
|
||||||
|
"legacy": false,
|
||||||
|
"queryType": "scan",
|
||||||
|
"resultFormat": "compactedList",
|
||||||
|
"virtualColumns": [
|
||||||
|
{
|
||||||
|
"expression": "timestamp_parse("timestamp",null,'UTC')",
|
||||||
|
"name": "v0",
|
||||||
|
"outputType": "LONG",
|
||||||
|
"type": "expression",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"type": "scan",
|
||||||
|
},
|
||||||
|
"shuffleCheckHasMultipleValues": true,
|
||||||
|
"shuffleSpec": {
|
||||||
|
"clusterBy": {
|
||||||
|
"bucketByCount": 1,
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"columnName": "__bucket",
|
||||||
|
"order": "ASCENDING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"columnName": "__boost",
|
||||||
|
"order": "ASCENDING",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"targetSize": 3000000,
|
||||||
|
"type": "targetSize",
|
||||||
|
},
|
||||||
|
"signature": [
|
||||||
|
{
|
||||||
|
"name": "__bucket",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "__boost",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "agent_type",
|
||||||
|
"type": "STRING",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "v0",
|
||||||
|
"type": "LONG",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"duration": 4056,
|
||||||
|
"output": "localStorage",
|
||||||
|
"phase": "FAILED",
|
||||||
|
"shuffle": "globalSort",
|
||||||
|
"sort": true,
|
||||||
|
"stageNumber": 0,
|
||||||
|
"startTime": "2024-07-26T18:05:02.399Z",
|
||||||
|
"workerCount": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"id": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4_1",
|
||||||
|
"input": [
|
||||||
|
{
|
||||||
|
"stage": 0,
|
||||||
|
"type": "stage",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"maxWorkerCount": 1,
|
||||||
|
"processor": {
|
||||||
|
"columnMappings": [
|
||||||
|
{
|
||||||
|
"outputColumn": "__time",
|
||||||
|
"queryColumn": "v0",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"outputColumn": "agent_type",
|
||||||
|
"queryColumn": "agent_type",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"dataSchema": {
|
||||||
|
"dataSource": "kttm-blank-lines",
|
||||||
|
"dimensionsSpec": {
|
||||||
|
"dimensionExclusions": [
|
||||||
|
"__time",
|
||||||
|
],
|
||||||
|
"dimensions": [
|
||||||
|
{
|
||||||
|
"createBitmapIndex": true,
|
||||||
|
"multiValueHandling": "SORTED_ARRAY",
|
||||||
|
"name": "agent_type",
|
||||||
|
"type": "string",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"includeAllDimensions": false,
|
||||||
|
"useSchemaDiscovery": false,
|
||||||
|
},
|
||||||
|
"granularitySpec": {
|
||||||
|
"intervals": [
|
||||||
|
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z",
|
||||||
|
],
|
||||||
|
"queryGranularity": {
|
||||||
|
"type": "none",
|
||||||
|
},
|
||||||
|
"rollup": false,
|
||||||
|
"type": "arbitrary",
|
||||||
|
},
|
||||||
|
"metricsSpec": [],
|
||||||
|
"timestampSpec": {
|
||||||
|
"column": "__time",
|
||||||
|
"format": "millis",
|
||||||
|
"missingValue": null,
|
||||||
|
},
|
||||||
|
"transformSpec": {
|
||||||
|
"filter": null,
|
||||||
|
"transforms": [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"tuningConfig": {
|
||||||
|
"maxNumWorkers": 1,
|
||||||
|
"maxRowsInMemory": 100000,
|
||||||
|
"rowsPerSegment": 3000000,
|
||||||
|
},
|
||||||
|
"type": "segmentGenerator",
|
||||||
|
},
|
||||||
|
"signature": [],
|
||||||
|
},
|
||||||
|
"stageNumber": 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"startTime": 2024-07-26T18:04:59.873Z,
|
||||||
"status": "FAILED",
|
"status": "FAILED",
|
||||||
"usageInfo": undefined,
|
"usageInfo": undefined,
|
||||||
"warnings": undefined,
|
"warnings": [
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"errorCode": "CannotParseExternalData",
|
||||||
|
"errorMessage": "Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)",
|
||||||
|
},
|
||||||
|
"exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)
|
||||||
|
at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)
|
||||||
|
at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)
|
||||||
|
at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)
|
||||||
|
at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)
|
||||||
|
at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)
|
||||||
|
at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)
|
||||||
|
at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)
|
||||||
|
at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)
|
||||||
|
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
|
||||||
|
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
|
||||||
|
at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)
|
||||||
|
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
|
||||||
|
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
|
||||||
|
at java.base/java.lang.Thread.run(Thread.java:840)
|
||||||
|
Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
|
||||||
|
at [Source: (byte[])""; line: 1, column: 0]
|
||||||
|
at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)
|
||||||
|
at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)
|
||||||
|
at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)
|
||||||
|
at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)
|
||||||
|
... 24 more
|
||||||
|
",
|
||||||
|
"host": "localhost:8101",
|
||||||
|
"stageNumber": 0,
|
||||||
|
"taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"errorCode": "CannotParseExternalData",
|
||||||
|
"errorMessage": "Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)",
|
||||||
|
},
|
||||||
|
"exceptionStackTrace": "org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)
|
||||||
|
at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:80)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:72)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$2.hasNext(CloseableIterator.java:93)
|
||||||
|
at org.apache.druid.java.util.common.parsers.CloseableIterator$1.hasNext(CloseableIterator.java:42)
|
||||||
|
at org.apache.druid.msq.input.external.ExternalSegment$1$1.hasNext(ExternalSegment.java:94)
|
||||||
|
at org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)
|
||||||
|
at org.apache.druid.segment.RowWalker.advance(RowWalker.java:75)
|
||||||
|
at org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)
|
||||||
|
at org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:374)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeededWithExceptionHandling(ScanQueryFrameProcessor.java:334)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:273)
|
||||||
|
at org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:88)
|
||||||
|
at org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:157)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:230)
|
||||||
|
at org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:138)
|
||||||
|
at org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:838)
|
||||||
|
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
|
||||||
|
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
|
||||||
|
at org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:259)
|
||||||
|
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
|
||||||
|
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
|
||||||
|
at java.base/java.lang.Thread.run(Thread.java:840)
|
||||||
|
Caused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
|
||||||
|
at [Source: (byte[])""; line: 1, column: 0]
|
||||||
|
at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)
|
||||||
|
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3609)
|
||||||
|
at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)
|
||||||
|
at org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)
|
||||||
|
at org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)
|
||||||
|
... 24 more
|
||||||
|
",
|
||||||
|
"host": "localhost:8101",
|
||||||
|
"stageNumber": 0,
|
||||||
|
"taskId": "query-ea3e36df-ad67-4870-b136-f5616b17d9c4-worker0_0",
|
||||||
|
},
|
||||||
|
],
|
||||||
}
|
}
|
||||||
`);
|
`);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -194,7 +194,7 @@ export interface ExecutionValue {
|
||||||
|
|
||||||
export class Execution {
|
export class Execution {
|
||||||
static USE_TASK_PAYLOAD = true;
|
static USE_TASK_PAYLOAD = true;
|
||||||
static USE_TASK_REPORTS = true;
|
static USE_TASK_REPORTS = false;
|
||||||
static INLINE_DATASOURCE_MARKER = '__query_select';
|
static INLINE_DATASOURCE_MARKER = '__query_select';
|
||||||
|
|
||||||
static getClusterCapacity: (() => Promise<CapacityInfo | undefined>) | undefined =
|
static getClusterCapacity: (() => Promise<CapacityInfo | undefined>) | undefined =
|
||||||
|
|
@ -235,7 +235,7 @@ export class Execution {
|
||||||
sqlQuery?: string,
|
sqlQuery?: string,
|
||||||
queryContext?: QueryContext,
|
queryContext?: QueryContext,
|
||||||
): Execution {
|
): Execution {
|
||||||
const { queryId, schema, result, errorDetails } = asyncSubmitResult;
|
const { queryId, schema, result, errorDetails, stages, counters, warnings } = asyncSubmitResult;
|
||||||
|
|
||||||
let queryResult: QueryResult | undefined;
|
let queryResult: QueryResult | undefined;
|
||||||
if (schema && result?.sampleRecords) {
|
if (schema && result?.sampleRecords) {
|
||||||
|
|
@ -263,6 +263,8 @@ export class Execution {
|
||||||
status: Execution.normalizeAsyncState(asyncSubmitResult.state),
|
status: Execution.normalizeAsyncState(asyncSubmitResult.state),
|
||||||
sqlQuery,
|
sqlQuery,
|
||||||
queryContext,
|
queryContext,
|
||||||
|
stages: Array.isArray(stages) && counters ? new Stages(stages, counters) : undefined,
|
||||||
|
warnings: Array.isArray(warnings) ? warnings : undefined,
|
||||||
error: executionError,
|
error: executionError,
|
||||||
destination:
|
destination:
|
||||||
typeof result?.dataSource === 'string'
|
typeof result?.dataSource === 'string'
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,8 @@ export interface StageDefinition {
|
||||||
startTime?: string;
|
startTime?: string;
|
||||||
duration?: number;
|
duration?: number;
|
||||||
sort?: boolean;
|
sort?: boolean;
|
||||||
|
shuffle?: string;
|
||||||
|
output?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClusterBy {
|
export interface ClusterBy {
|
||||||
|
|
@ -169,7 +171,7 @@ export type SegmentGenerationProgressFields =
|
||||||
| 'rowsPushed';
|
| 'rowsPushed';
|
||||||
|
|
||||||
export interface WarningCounter {
|
export interface WarningCounter {
|
||||||
type: 'warning';
|
type: 'warnings';
|
||||||
CannotParseExternalData?: number;
|
CannotParseExternalData?: number;
|
||||||
// More types of warnings might be added later
|
// More types of warnings might be added later
|
||||||
}
|
}
|
||||||
|
|
@ -192,6 +194,8 @@ function zeroChannelFields(): Record<ChannelFields, number> {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type Counters = Record<string, Record<string, StageWorkerCounter>>;
|
||||||
|
|
||||||
export class Stages {
|
export class Stages {
|
||||||
static readonly QUERY_START_FACTOR = 0.05;
|
static readonly QUERY_START_FACTOR = 0.05;
|
||||||
static readonly QUERY_END_FACTOR = 0.05;
|
static readonly QUERY_END_FACTOR = 0.05;
|
||||||
|
|
@ -205,12 +209,9 @@ export class Stages {
|
||||||
}
|
}
|
||||||
|
|
||||||
public readonly stages: StageDefinition[];
|
public readonly stages: StageDefinition[];
|
||||||
private readonly counters?: Record<string, Record<string, StageWorkerCounter>>;
|
private readonly counters?: Counters;
|
||||||
|
|
||||||
constructor(
|
constructor(stages: StageDefinition[], counters?: Counters) {
|
||||||
stages: StageDefinition[],
|
|
||||||
counters?: Record<string, Record<string, StageWorkerCounter>>,
|
|
||||||
) {
|
|
||||||
this.stages = stages;
|
this.stages = stages;
|
||||||
this.counters = counters;
|
this.counters = counters;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
import { C } from '@druid-toolkit/query';
|
import { C } from '@druid-toolkit/query';
|
||||||
|
|
||||||
import type { StageDefinition } from '../stages/stages';
|
import type { Counters, StageDefinition } from '../stages/stages';
|
||||||
|
|
||||||
export type TaskStatus = 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS';
|
export type TaskStatus = 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS';
|
||||||
export type TaskStatusWithCanceled = TaskStatus | 'CANCELED';
|
export type TaskStatusWithCanceled = TaskStatus | 'CANCELED';
|
||||||
|
|
@ -112,7 +112,7 @@ export interface MsqTaskReportResponse {
|
||||||
segmentLoadWaiterStatus?: SegmentLoadWaiterStatus;
|
segmentLoadWaiterStatus?: SegmentLoadWaiterStatus;
|
||||||
};
|
};
|
||||||
stages: StageDefinition[];
|
stages: StageDefinition[];
|
||||||
counters: Record<string, Record<string, any>>;
|
counters: Counters;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
error?: any;
|
error?: any;
|
||||||
|
|
|
||||||
|
|
@ -151,11 +151,12 @@ export class WorkbenchQuery {
|
||||||
return WorkbenchQuery.enabledQueryEngines;
|
return WorkbenchQuery.enabledQueryEngines;
|
||||||
}
|
}
|
||||||
|
|
||||||
static fromEffectiveQueryAndContext(queryString: string, context: QueryContext): WorkbenchQuery {
|
static fromTaskQueryAndContext(queryString: string, context: QueryContext): WorkbenchQuery {
|
||||||
const noSqlOuterLimit = typeof context['sqlOuterLimit'] === 'undefined';
|
const noSqlOuterLimit = typeof context['sqlOuterLimit'] === 'undefined';
|
||||||
const cleanContext = deleteKeys(context, ['sqlOuterLimit']);
|
const cleanContext = deleteKeys(context, ['sqlOuterLimit']);
|
||||||
|
|
||||||
let retQuery = WorkbenchQuery.blank()
|
let retQuery = WorkbenchQuery.blank()
|
||||||
|
.changeEngine('sql-msq-task')
|
||||||
.changeQueryString(queryString)
|
.changeQueryString(queryString)
|
||||||
.changeQueryContext(cleanContext);
|
.changeQueryContext(cleanContext);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -184,7 +184,7 @@ export async function getTaskExecution(
|
||||||
|
|
||||||
if (!execution) {
|
if (!execution) {
|
||||||
const statusResp = await Api.instance.get<AsyncStatusResponse>(
|
const statusResp = await Api.instance.get<AsyncStatusResponse>(
|
||||||
`/druid/v2/sql/statements/${encodedId}`,
|
`/druid/v2/sql/statements/${encodedId}?detail=true`,
|
||||||
{
|
{
|
||||||
cancelToken,
|
cancelToken,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,10 @@ import { formatForFormat } from './download';
|
||||||
|
|
||||||
describe('download', () => {
|
describe('download', () => {
|
||||||
it('.formatForFormat', () => {
|
it('.formatForFormat', () => {
|
||||||
expect(formatForFormat(null, 'csv')).toEqual('"null"');
|
expect(formatForFormat(null, 'csv')).toEqual('');
|
||||||
|
expect(formatForFormat(null, 'tsv')).toEqual('');
|
||||||
|
expect(formatForFormat('', 'csv')).toEqual('""');
|
||||||
|
expect(formatForFormat('null', 'csv')).toEqual('"null"');
|
||||||
expect(formatForFormat('hello\nworld', 'csv')).toEqual('"hello world"');
|
expect(formatForFormat('hello\nworld', 'csv')).toEqual('"hello world"');
|
||||||
expect(formatForFormat(123, 'csv')).toEqual('"123"');
|
expect(formatForFormat(123, 'csv')).toEqual('"123"');
|
||||||
expect(formatForFormat(new Date('2021-01-02T03:04:05.678Z'), 'csv')).toEqual(
|
expect(formatForFormat(new Date('2021-01-02T03:04:05.678Z'), 'csv')).toEqual(
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,8 @@ export function downloadUrl(url: string, filename: string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
export function formatForFormat(s: null | string | number | Date, format: 'csv' | 'tsv'): string {
|
export function formatForFormat(s: null | string | number | Date, format: 'csv' | 'tsv'): string {
|
||||||
|
if (s == null) return '';
|
||||||
|
|
||||||
// stringify and remove line break
|
// stringify and remove line break
|
||||||
const str = stringifyValue(s).replace(/(?:\r\n|\r|\n)/g, ' ');
|
const str = stringifyValue(s).replace(/(?:\r\n|\r|\n)/g, ' ');
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -173,7 +173,7 @@ LIMIT 100`,
|
||||||
}
|
}
|
||||||
|
|
||||||
onNewTab(
|
onNewTab(
|
||||||
WorkbenchQuery.fromEffectiveQueryAndContext(
|
WorkbenchQuery.fromTaskQueryAndContext(
|
||||||
execution.sqlQuery,
|
execution.sqlQuery,
|
||||||
execution.queryContext,
|
execution.queryContext,
|
||||||
).changeLastExecution({ engine: 'sql-msq-task', id: w.taskId }),
|
).changeLastExecution({ engine: 'sql-msq-task', id: w.taskId }),
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,8 @@ import type { SqlQuery } from '@druid-toolkit/query';
|
||||||
import { SqlExpression } from '@druid-toolkit/query';
|
import { SqlExpression } from '@druid-toolkit/query';
|
||||||
import classNames from 'classnames';
|
import classNames from 'classnames';
|
||||||
import copy from 'copy-to-clipboard';
|
import copy from 'copy-to-clipboard';
|
||||||
import React, { ComponentProps } from 'react';
|
import type { ComponentProps } from 'react';
|
||||||
|
import React from 'react';
|
||||||
|
|
||||||
import { SpecDialog, StringInputDialog } from '../../dialogs';
|
import { SpecDialog, StringInputDialog } from '../../dialogs';
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -464,7 +465,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
|
||||||
}
|
}
|
||||||
|
|
||||||
this.handleNewTab(
|
this.handleNewTab(
|
||||||
WorkbenchQuery.fromEffectiveQueryAndContext(
|
WorkbenchQuery.fromTaskQueryAndContext(
|
||||||
execution.sqlQuery,
|
execution.sqlQuery,
|
||||||
execution.queryContext,
|
execution.queryContext,
|
||||||
).changeLastExecution({ engine: 'sql-msq-task', id: taskId }),
|
).changeLastExecution({ engine: 'sql-msq-task', id: taskId }),
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue