Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: use EXTEND syntax (#13985) 

* use EXTEND syntax

* update licenses

* update demo queries

* updated snapshots

* add join algorithm selector

* dismiss
This commit is contained in:
Vadim Ogievetsky 2023-03-29 16:19:49 -07:00 committed by GitHub
parent f715887172
commit abb7133153
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 1388 additions and 1275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
licenses.yaml
View File

@ -5080,7 +5080,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Sebastian McKenzie
version: 7.5.5
version: 7.12.11
license_file_path: licenses/bin/@babel-code-frame.MIT
---
@ -5099,8 +5099,8 @@ name: "@babel/helper-validator-identifier"
license_category: binary
module: web-console
license_name: MIT License
copyright: Sebastian McKenzie and other contributors
version: 7.14.0
copyright: The Babel Team
version: 7.19.1
license_file_path: licenses/bin/@babel-helper-validator-identifier.MIT
---
@ -5109,8 +5109,8 @@ name: "@babel/highlight"
license_category: binary
module: web-console
license_name: MIT License
copyright: suchipi
version: 7.5.0
copyright: The Babel Team
version: 7.18.6
license_file_path: licenses/bin/@babel-highlight.MIT
---
@ -5120,7 +5120,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: The Babel Team
version: 7.14.8
version: 7.21.0
license_file_path: licenses/bin/@babel-runtime.MIT
---
@ -5140,7 +5140,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Palantir Technologies
version: 4.0.1
version: 4.1.19
---
@ -5149,7 +5149,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Palantir Technologies
version: 4.0.3
version: 4.17.6
---
@ -5158,7 +5158,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Palantir Technologies
version: 4.0.3
version: 4.4.25
---
@ -5167,7 +5167,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Palantir Technologies
version: 4.1.1
version: 4.14.3
---
@ -5176,7 +5176,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Palantir Technologies
version: 1.0.3
version: 1.13.10
---
@ -5285,7 +5285,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Juggle
version: 3.3.1
version: 3.4.0
---
@ -5294,7 +5294,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Federico Zivolo
version: 2.11.4
version: 2.11.6
license_file_path: licenses/bin/@popperjs-core.MIT
---
@ -5319,6 +5319,36 @@ license_file_path: licenses/bin/@types-parse-json.MIT
---
name: "@types/prop-types"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 15.7.3
license_file_path: licenses/bin/@types-prop-types.MIT
---
name: "@types/react"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 16.14.35
license_file_path: licenses/bin/@types-react.MIT
---
name: "@types/scheduler"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 0.16.1
license_file_path: licenses/bin/@types-scheduler.MIT
---
name: "ace-builds"
license_category: binary
module: web-console
@ -5379,6 +5409,16 @@ license_file_path: licenses/bin/babel-plugin-syntax-jsx.MIT
---
name: "call-bind"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.2
license_file_path: licenses/bin/call-bind.MIT
---
name: "callsites"
license_category: binary
module: web-console
@ -5434,7 +5474,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Jed Watson
version: 2.2.6
version: 2.3.2
license_file_path: licenses/bin/classnames.MIT
---
@ -5634,7 +5674,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.1.3
version: 1.2.0
license_file_path: licenses/bin/define-properties.MIT
---
@ -5693,7 +5733,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Imply Data
version: 0.17.5
version: 0.18.2
---
@ -5727,16 +5767,6 @@ license_file_path: licenses/bin/escape-string-regexp.MIT
---
name: "esutils"
license_category: binary
module: web-console
license_name: BSD-2-Clause License
copyright: Yusuke Suzuki
version: 2.0.3
license_file_path: licenses/bin/esutils.BSD2
---
name: "file-saver"
license_category: binary
module: web-console
@ -5787,6 +5817,26 @@ license_file_path: licenses/bin/function-bind.MIT
---
name: "functions-have-names"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.2.3
license_file_path: licenses/bin/functions-have-names.MIT
---
name: "get-intrinsic"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.2.0
license_file_path: licenses/bin/get-intrinsic.MIT
---
name: "gud"
license_category: binary
module: web-console
@ -5817,6 +5867,36 @@ license_file_path: licenses/bin/has-own-prop.MIT
---
name: "has-property-descriptors"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/has-property-descriptors.MIT
---
name: "has-symbols"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.3
license_file_path: licenses/bin/has-symbols.MIT
---
name: "has-tostringtag"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/has-tostringtag.MIT
---
name: "has"
license_category: binary
module: web-console
@ -5907,6 +5987,16 @@ license_file_path: licenses/bin/is-arrayish.MIT
---
name: "is-core-module"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 2.10.0
license_file_path: licenses/bin/is-core-module.MIT
---
name: "is-date-object"
license_category: binary
module: web-console
@ -5922,7 +6012,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.4
version: 1.1.4
license_file_path: licenses/bin/is-regex.MIT
---
@ -6341,7 +6431,7 @@ license_category: binary
module: web-console
license_name: BSD-3-Clause License
copyright: React Community
version: 4.4.2
version: 4.4.5
license_file_path: licenses/bin/react-transition-group.BSD3
---
@ -6361,7 +6451,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Ben Newman
version: 0.13.7
version: 0.13.11
license_file_path: licenses/bin/regenerator-runtime.MIT
---
@ -6371,7 +6461,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.2.0
version: 1.4.3
license_file_path: licenses/bin/regexp.prototype.flags.MIT
---
@ -6401,7 +6491,7 @@ license_category: binary
module: web-console
license_name: MIT License
copyright: James Halliday
version: 1.12.0
version: 1.22.1
license_file_path: licenses/bin/resolve.MIT
---
@ -6466,6 +6556,16 @@ license_file_path: licenses/bin/supports-color.MIT
---
name: "supports-preserve-symlinks-flag"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/supports-preserve-symlinks-flag.MIT
---
name: "tiny-invariant"
license_category: binary
module: web-console

21
licenses/bin/@types-prop-types.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/@types-react.MIT Executable file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/@types-scheduler.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/call-bind.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

2
licenses/bin/classnames.MIT
View File

@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2017 Jed Watson
Copyright (c) 2018 Jed Watson
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

21
licenses/bin/functions-have-names.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/get-intrinsic.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-property-descriptors.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-symbols.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2016 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-tostringtag.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

20
licenses/bin/is-core-module.MIT Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Dave Justice
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

3
licenses/bin/regexp.prototype.flags.MIT
View File

@ -18,4 +18,5 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
THE SOFTWARE.

21
licenses/bin/supports-preserve-symlinks-flag.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

5
web-console/e2e-tests/multi-stage-query.spec.ts
View File

@ -52,10 +52,9 @@ FROM TABLE(
'{"type":"local","filter":"wikiticker-2015-09-12-sampled.json.gz","baseDir":${JSON.stringify(
DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
)}}',
'{"type":"json"}',
'[{"name":"channel","type":"string"}]'
'{"type":"json"}'
)
))
) EXTEND (channel VARCHAR))
SELECT
channel,
CAST(COUNT(*) AS VARCHAR) AS "CountString"

14
web-console/package-lock.json generated
View File

@ -22,7 +22,7 @@
"d3-axis": "^2.1.0",
"d3-scale": "^3.3.0",
"d3-selection": "^2.0.0",
"druid-query-toolkit": "^0.17.5",
"druid-query-toolkit": "^0.18.2",
"file-saver": "^2.0.2",
"follow-redirects": "^1.14.7",
"fontsource-open-sans": "^3.0.9",
@ -8211,9 +8211,9 @@
}
},
"node_modules/druid-query-toolkit": {
"version": "0.17.5",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz",
"integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==",
"version": "0.18.2",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz",
"integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==",
"dependencies": {
"tslib": "^2.3.1"
},
@ -32625,9 +32625,9 @@
}
},
"druid-query-toolkit": {
"version": "0.17.5",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz",
"integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==",
"version": "0.18.2",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz",
"integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==",
"requires": {
"tslib": "^2.3.1"
}

5
web-console/package.json
View File

@ -17,9 +17,6 @@
"testMatch": [
"**/?(*.)+(spec).ts?(x)"
],
"snapshotSerializers": [
"enzyme-to-json/serializer"
],
"setupFilesAfterEnv": [
"<rootDir>src/setup-tests.ts"
]
@ -79,7 +76,7 @@
"d3-axis": "^2.1.0",
"d3-scale": "^3.3.0",
"d3-selection": "^2.0.0",
"druid-query-toolkit": "^0.17.5",
"druid-query-toolkit": "^0.18.2",
"file-saver": "^2.0.2",
"follow-redirects": "^1.14.7",
"fontsource-open-sans": "^3.0.9",

11
web-console/script/create-sql-docs.js
View File

@ -23,9 +23,16 @@ const snarkdown = require('snarkdown');
const writefile = 'lib/sql-docs.js';
const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162;
const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 164;
const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
const initialFunctionDocs = {
TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an external.')]],
EXTERN: [
['inputSource, inputFormat, rowSignature?', convertMarkdownToHtml('Reads external data')],
],
};
function hasHtmlTags(str) {
return /<(a|br|span|div|p|code)\/?>/.test(str);
}
@ -67,7 +74,7 @@ const readDoc = async () => {
const lines = data.split('\n');
const functionDocs = {};
const functionDocs = initialFunctionDocs;
const dataTypeDocs = {};
for (let line of lines) {
const functionMatch = line.match(/^\|\s*`(\w+)\(([^|]*)\)`\s*\|([^|]+)\|(?:([^|]+)\|)?$/);

4
web-console/src/bootstrap/ace.scss
View File

@ -35,6 +35,10 @@
text-decoration: underline;
text-decoration-style: wavy;
}
.ace_storage.ace_type {
color: #27c923;
}
// END: Custom code styles
&.no-background {

88
web-console/src/druid-models/execution/execution-ingest-complete.mock.ts
View File

@ -19,26 +19,33 @@
import { Execution } from './execution';
/*
For query:
===== Query =====
REPLACE INTO "kttm_simple" OVERWRITE ALL
SELECT TIME_PARSE("timestamp") AS "__time", agent_type
SELECT
TIME_PARSE("timestamp") AS "__time",
"agent_type"
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]'
'{"type":"json"}'
)
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY ALL TIME
===== Context =====
{
"maxNumTasks": 2
}
*/
export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{
task: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9',
task: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
payload: {
type: 'query_controller',
id: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9',
id: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
spec: {
query: {
queryType: 'scan',
@ -78,10 +85,11 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
finalize: false,
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
maxNumTasks: 2,
queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
sqlReplaceTimeChunks: 'all',
},
granularity: { type: 'all' },
@ -100,22 +108,23 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 },
},
sqlQuery:
'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL TIME',
'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME',
sqlQueryContext: {
finalizeAggregations: false,
maxParseExceptions: 0,
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
groupByEnableMultiValueUnnesting: false,
sqlInsertSegmentGranularity: '{"type":"all"}',
maxNumTasks: 2,
sqlReplaceTimeChunks: 'all',
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
},
sqlTypeNames: ['TIMESTAMP', 'VARCHAR'],
context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true },
groupId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9',
groupId: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
dataSource: 'kttm_simple',
resource: {
availabilityGroup: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9',
availabilityGroup: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
requiredCapacity: 1,
},
},
@ -124,12 +133,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{
multiStageQuery: {
type: 'multiStageQuery',
taskId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9',
taskId: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
payload: {
status: {
status: 'SUCCESS',
startTime: '2022-10-31T16:11:13.160Z',
durationMs: 9012,
startTime: '2023-03-27T22:17:02.401Z',
durationMs: 28854,
pendingTasks: 0,
runningTasks: 2,
},
@ -137,7 +146,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{
stageNumber: 0,
definition: {
id: 'b94caff0-f693-47ed-a242-ccf837550383_0',
id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0',
input: [
{
type: 'external',
@ -184,11 +193,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
finalize: false,
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
maxNumTasks: 2,
queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
scanSignature:
'[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9',
sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
sqlReplaceTimeChunks: 'all',
},
granularity: { type: 'all' },
@ -201,23 +211,24 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
],
shuffleSpec: {
type: 'targetSize',
clusterBy: { columns: [{ columnName: '__boost' }] },
clusterBy: { columns: [{ columnName: '__boost', order: 'ASCENDING' }] },
targetSize: 3000000,
},
maxWorkerCount: 1,
shuffleCheckHasMultipleValues: true,
maxInputBytesPerWorker: 10737418240,
},
phase: 'FINISHED',
workerCount: 1,
partitionCount: 1,
startTime: '2022-10-31T16:11:15.380Z',
duration: 4887,
startTime: '2023-03-27T22:17:02.792Z',
duration: 24236,
sort: true,
},
{
stageNumber: 1,
definition: {
id: 'b94caff0-f693-47ed-a242-ccf837550383_1',
id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1',
input: [{ type: 'stage', stage: 0 }],
processor: {
type: 'segmentGenerator',
@ -235,6 +246,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
],
dimensionExclusions: ['__time'],
includeAllDimensions: false,
useSchemaDiscovery: false,
},
metricsSpec: [],
granularitySpec: {
@ -257,32 +269,48 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
},
signature: [],
maxWorkerCount: 1,
maxInputBytesPerWorker: 10737418240,
},
phase: 'FINISHED',
workerCount: 1,
partitionCount: 1,
startTime: '2022-10-31T16:11:20.264Z',
duration: 1908,
startTime: '2023-03-27T22:17:26.978Z',
duration: 4276,
},
],
counters: {
'0': {
'0': {
input0: { type: 'channel', rows: [465346], files: [1], totalFiles: [1] },
input0: {
type: 'channel',
rows: [465346],
bytes: [360464067],
files: [1],
totalFiles: [1],
},
output: { type: 'channel', rows: [465346], bytes: [25430674], frames: [4] },
shuffle: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] },
sortProgress: {
type: 'sortProgress',
totalMergingLevels: 3,
levelToTotalBatches: { '0': 2, '1': 1, '2': 1 },
levelToMergedBatches: { '0': 2, '1': 1, '2': 1 },
levelToTotalBatches: { '0': 1, '1': 1, '2': 1 },
levelToMergedBatches: { '0': 1, '1': 1, '2': 1 },
totalMergersForUltimateLevel: 1,
progressDigest: 1.0,
},
},
},
'1': {
'0': { input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] } },
'0': {
input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] },
segmentGenerationProgress: {
type: 'segmentGenerationProgress',
rowsProcessed: 465346,
rowsPersisted: 465346,
rowsMerged: 465346,
rowsPushed: 465346,
},
},
},
},
},

165
web-console/src/druid-models/execution/execution-ingest-error.mock.ts
View File

@ -21,30 +21,32 @@ import { Execution } from './execution';
/*
===== Query =====
REPLACE INTO "kttm_simple" OVERWRITE ALL
SELECT TIME_PARSE("timestamp") AS "__time", agent_type
REPLACE INTO "kttm-blank-lines" OVERWRITE ALL
SELECT
TIME_PARSE("timestamp") AS "__time",
"agent_type"
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]'
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}',
'{"type":"json"}'
)
)
PARTITIONED BY ALL
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY DAY
===== Context =====
{
"maxParseExceptions": 10
"maxParseExceptions": 2,
"maxNumTasks": 2
}
*/
export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{
task: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
task: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
payload: {
type: 'query_controller',
id: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
id: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
spec: {
query: {
queryType: 'scan',
@ -52,7 +54,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
type: 'external',
inputSource: {
type: 'http',
uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'],
uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'],
},
inputFormat: {
type: 'json',
@ -84,11 +86,12 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
finalize: false,
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 10,
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
maxNumTasks: 2,
maxParseExceptions: 2,
queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
sqlInsertSegmentGranularity: '"DAY"',
sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
sqlReplaceTimeChunks: 'all',
},
granularity: { type: 'all' },
@ -99,30 +102,31 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
],
destination: {
type: 'dataSource',
dataSource: 'kttm_simple',
segmentGranularity: { type: 'all' },
dataSource: 'kttm-blank-lines',
segmentGranularity: 'DAY',
replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
},
assignmentStrategy: 'max',
tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 },
},
sqlQuery:
'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL',
'REPLACE INTO "kttm-blank-lines" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY DAY',
sqlQueryContext: {
maxParseExceptions: 10,
maxParseExceptions: 2,
finalizeAggregations: false,
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
groupByEnableMultiValueUnnesting: false,
sqlInsertSegmentGranularity: '{"type":"all"}',
sqlInsertSegmentGranularity: '"DAY"',
maxNumTasks: 2,
sqlReplaceTimeChunks: 'all',
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
},
sqlTypeNames: ['TIMESTAMP', 'VARCHAR'],
context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true },
groupId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
dataSource: 'kttm_simple',
groupId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
dataSource: 'kttm-blank-lines',
resource: {
availabilityGroup: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
availabilityGroup: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
requiredCapacity: 1,
},
},
@ -131,85 +135,48 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{
multiStageQuery: {
type: 'multiStageQuery',
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
payload: {
status: {
status: 'FAILED',
errorReport: {
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost',
error: {
errorCode: 'TooManyWarnings',
maxWarnings: 10,
maxWarnings: 2,
rootErrorCode: 'CannotParseExternalData',
errorMessage:
'Too many warnings of type CannotParseExternalData generated (max = 10)',
errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)',
},
},
warnings: [
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost:8101',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)',
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost:8101',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)',
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
],
startTime: '2022-10-31T16:16:22.464Z',
durationMs: 7229,
startTime: '2023-03-27T22:11:24.945Z',
durationMs: 14106,
pendingTasks: 0,
runningTasks: 2,
},
@ -217,13 +184,15 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{
stageNumber: 0,
definition: {
id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_0',
id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_0',
input: [
{
type: 'external',
inputSource: {
type: 'http',
uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'],
uris: [
'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json',
],
},
inputFormat: {
type: 'json',
@ -262,45 +231,54 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
finalize: false,
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 10,
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
maxNumTasks: 2,
maxParseExceptions: 2,
queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
scanSignature:
'[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a',
sqlInsertSegmentGranularity: '"DAY"',
sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
sqlReplaceTimeChunks: 'all',
},
granularity: { type: 'all' },
},
},
signature: [
{ name: '__bucket', type: 'LONG' },
{ name: '__boost', type: 'LONG' },
{ name: 'agent_type', type: 'STRING' },
{ name: 'v0', type: 'LONG' },
],
shuffleSpec: {
type: 'targetSize',
clusterBy: { columns: [{ columnName: '__boost' }] },
clusterBy: {
columns: [
{ columnName: '__bucket', order: 'ASCENDING' },
{ columnName: '__boost', order: 'ASCENDING' },
],
bucketByCount: 1,
},
targetSize: 3000000,
},
maxWorkerCount: 1,
shuffleCheckHasMultipleValues: true,
maxInputBytesPerWorker: 10737418240,
},
phase: 'FAILED',
workerCount: 1,
startTime: '2022-10-31T16:16:24.680Z',
duration: 5013,
startTime: '2023-03-27T22:11:25.310Z',
duration: 13741,
sort: true,
},
{
stageNumber: 1,
definition: {
id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_1',
id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_1',
input: [{ type: 'stage', stage: 0 }],
processor: {
type: 'segmentGenerator',
dataSchema: {
dataSource: 'kttm_simple',
dataSource: 'kttm-blank-lines',
timestampSpec: { column: '__time', format: 'millis', missingValue: null },
dimensionsSpec: {
dimensions: [
@ -313,6 +291,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
],
dimensionExclusions: ['__time'],
includeAllDimensions: false,
useSchemaDiscovery: false,
},
metricsSpec: [],
granularitySpec: {
@ -335,22 +314,24 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
},
signature: [],
maxWorkerCount: 1,
maxInputBytesPerWorker: 10737418240,
},
},
],
counters: {
'0': {
'0': {
input0: { type: 'channel', rows: [237883], totalFiles: [1] },
output: { type: 'channel', rows: [141660], bytes: [7685544], frames: [1] },
input0: { type: 'channel', rows: [10], bytes: [7658], files: [1], totalFiles: [1] },
output: { type: 'channel', rows: [10], bytes: [712], frames: [1] },
sortProgress: {
type: 'sortProgress',
totalMergingLevels: -1,
levelToTotalBatches: {},
totalMergingLevels: 3,
levelToTotalBatches: { '0': 1, '1': 1, '2': -1 },
levelToMergedBatches: {},
totalMergersForUltimateLevel: -1,
progressDigest: 0.0,
},
warnings: { type: 'warnings', CannotParseExternalData: 17 },
warnings: { type: 'warnings', CannotParseExternalData: 3 },
},
},
},

83
web-console/src/druid-models/execution/execution.spec.ts
View File

@ -43,10 +43,10 @@ describe('Execution', () => {
"useLineageBasedSegmentAllocation": true,
},
"dataSource": "kttm_simple",
"groupId": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9",
"id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9",
"groupId": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"id": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"resource": Object {
"availabilityGroup": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9",
"availabilityGroup": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"requiredCapacity": 1,
},
"spec": Object {
@ -80,10 +80,11 @@ describe('Execution', () => {
"finalize": false,
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all",
},
"dataSource": Object {
@ -139,22 +140,24 @@ describe('Execution', () => {
},
},
"sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL
SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
\\"agent_type\\"
FROM TABLE(
EXTERN(
'{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}',
'{\\"type\\":\\"json\\"}',
'[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]'
'{\\"type\\":\\"json\\"}'
)
)
) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR)
PARTITIONED BY ALL TIME",
"sqlQueryContext": Object {
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 2,
"maxParseExceptions": 0,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all",
},
"sqlTypeNames": Array [
@ -163,7 +166,7 @@ describe('Execution', () => {
],
"type": "query_controller",
},
"task": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9",
"task": "query-b55f3432-7810-4529-80ed-780a926a6f03",
},
"capacityInfo": undefined,
"destination": Object {
@ -176,10 +179,10 @@ describe('Execution', () => {
},
"type": "dataSource",
},
"duration": 9012,
"duration": 28854,
"engine": "sql-msq-task",
"error": undefined,
"id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9",
"id": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"nativeQuery": Object {
"columns": Array [
"agent_type",
@ -189,10 +192,11 @@ describe('Execution', () => {
"finalize": false,
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all",
},
"dataSource": Object {
@ -244,24 +248,29 @@ describe('Execution', () => {
"queryContext": Object {
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 2,
"maxParseExceptions": 0,
},
"result": undefined,
"sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL
SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
\\"agent_type\\"
FROM TABLE(
EXTERN(
'{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}',
'{\\"type\\":\\"json\\"}',
'[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]'
'{\\"type\\":\\"json\\"}'
)
)
) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR)
PARTITIONED BY ALL TIME",
"stages": Stages {
"counters": Object {
"0": Object {
"0": Object {
"input0": Object {
"bytes": Array [
360464067,
],
"files": Array [
1,
],
@ -299,12 +308,12 @@ describe('Execution', () => {
},
"sortProgress": Object {
"levelToMergedBatches": Object {
"0": 2,
"0": 1,
"1": 1,
"2": 1,
},
"levelToTotalBatches": Object {
"0": 2,
"0": 1,
"1": 1,
"2": 1,
},
@ -329,13 +338,20 @@ describe('Execution', () => {
],
"type": "channel",
},
"segmentGenerationProgress": Object {
"rowsMerged": 465346,
"rowsPersisted": 465346,
"rowsProcessed": 465346,
"rowsPushed": 465346,
"type": "segmentGenerationProgress",
},
},
},
},
"stages": Array [
Object {
"definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_0",
"id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0",
"input": Array [
Object {
"inputFormat": Object {
@ -363,6 +379,7 @@ describe('Execution', () => {
"type": "external",
},
],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1,
"processor": Object {
"query": Object {
@ -375,10 +392,11 @@ describe('Execution', () => {
"finalize": false,
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all",
},
"dataSource": Object {
@ -414,6 +432,7 @@ describe('Execution', () => {
"columns": Array [
Object {
"columnName": "__boost",
"order": "ASCENDING",
},
],
},
@ -435,23 +454,24 @@ describe('Execution', () => {
},
],
},
"duration": 4887,
"duration": 24236,
"partitionCount": 1,
"phase": "FINISHED",
"sort": true,
"stageNumber": 0,
"startTime": "2022-10-31T16:11:15.380Z",
"startTime": "2023-03-27T22:17:02.792Z",
"workerCount": 1,
},
Object {
"definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_1",
"id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1",
"input": Array [
Object {
"stage": 0,
"type": "stage",
},
],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1,
"processor": Object {
"columnMappings": Array [
@ -479,6 +499,7 @@ describe('Execution', () => {
},
],
"includeAllDimensions": false,
"useSchemaDiscovery": false,
},
"granularitySpec": Object {
"intervals": Array [
@ -510,16 +531,16 @@ describe('Execution', () => {
},
"signature": Array [],
},
"duration": 1908,
"duration": 4276,
"partitionCount": 1,
"phase": "FINISHED",
"stageNumber": 1,
"startTime": "2022-10-31T16:11:20.264Z",
"startTime": "2023-03-27T22:17:26.978Z",
"workerCount": 1,
},
],
},
"startTime": 2022-10-31T16:11:13.160Z,
"startTime": 2023-03-27T22:17:02.401Z,
"status": "SUCCESS",
"usageInfo": Object {
"pendingTasks": 0,

44
web-console/src/druid-models/external-config/external-config.ts
View File

@ -22,10 +22,12 @@ import {
F,
filterMap,
L,
SqlColumnDeclaration,
SqlExpression,
SqlFunction,
SqlLiteral,
SqlStar,
SqlType,
} from 'druid-query-toolkit';
import * as JSONBig from 'json-bigint-native';
@ -46,12 +48,7 @@ function joinLinesMax(lines: string[], max: number) {
export interface ExternalConfig {
inputSource: InputSource;
inputFormat: InputFormat;
signature: SignatureColumn[];
}
export interface SignatureColumn {
name: string;
type: string;
signature: readonly SqlColumnDeclaration[];
}
export function summarizeInputSource(inputSource: InputSource, multiline: boolean): string {
@ -123,10 +120,9 @@ export function externalConfigToTableExpression(config: ExternalConfig): SqlExpr
return SqlExpression.parse(`TABLE(
EXTERN(
${L(JSONBig.stringify(config.inputSource))},
${L(JSONBig.stringify(config.inputFormat))},
${L(JSONBig.stringify(config.signature))}
${L(JSONBig.stringify(config.inputFormat))}
)
)`);
) EXTEND (${config.signature.join(', ')})`);
}
export function externalConfigToInitDimensions(
@ -136,9 +132,10 @@ export function externalConfigToInitDimensions(
): SqlExpression[] {
return (timeExpression ? [timeExpression.as('__time')] : [])
.concat(
filterMap(config.signature, ({ name }, i) => {
if (timeExpression && timeExpression.containsColumnName(name)) return;
return C(name).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(name) as any);
filterMap(config.signature, (columnDeclaration, i) => {
const columnName = columnDeclaration.getColumnName();
if (timeExpression && timeExpression.containsColumnName(columnName)) return;
return C(columnName).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(columnName) as any);
}),
)
.slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS);
@ -150,12 +147,12 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig {
}
const tableFn = query.fromClause?.expressions?.first();
if (!(tableFn instanceof SqlFunction) || tableFn.functionName !== 'TABLE') {
if (!(tableFn instanceof SqlFunction) || tableFn.getEffectiveFunctionName() !== 'TABLE') {
throw new Error(`External FROM must be a TABLE function`);
}
const externFn = tableFn.getArg(0);
if (!(externFn instanceof SqlFunction) || externFn.functionName !== 'EXTERN') {
if (!(externFn instanceof SqlFunction) || externFn.getEffectiveFunctionName() !== 'EXTERN') {
throw new Error(`Within the TABLE function there must be an extern function`);
}
@ -175,12 +172,19 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig {
throw new Error(`The second argument to the extern function must be a string embedding JSON`);
}
let signature: any;
try {
const arg2 = externFn.getArg(2);
signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#');
} catch {
throw new Error(`The third argument to the extern function must be a string embedding JSON`);
let signature: readonly SqlColumnDeclaration[];
const columnDeclarations = tableFn.getColumnDeclarations();
if (columnDeclarations) {
signature = columnDeclarations;
} else {
try {
const arg2 = externFn.getArg(2);
signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#').map(
(c: any) => SqlColumnDeclaration.create(c.name, SqlType.fromNativeType(c.type)),
);
} catch {
throw new Error(`The third argument to the extern function must be a string embedding JSON`);
}
}
return {

5
web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
View File

@ -28,10 +28,9 @@ describe('ingest-query-pattern', () => {
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://example.com/data.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"json"}'
)
))
) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" BIGINT, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" BIGINT, "window" VARCHAR))
SELECT
TIME_PARSE("timestamp") AS __time,
agent_category,

2
web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
View File

@ -133,7 +133,7 @@ function verifyHasOutputName(expression: SqlExpression): void {
}
export function fitIngestQueryPattern(query: SqlQuery): IngestQueryPattern {
if (query.explainClause) throw new Error(`Can not use EXPLAIN in the data loader flow`);
if (query.explain) throw new Error(`Can not use EXPLAIN in the data loader flow`);
if (query.havingClause) throw new Error(`Can not use HAVING in the data loader flow`);
if (query.orderByClause) throw new Error(`Can not USE ORDER BY in the data loader flow`);
if (query.limitClause) throw new Error(`Can not use LIMIT in the data loader flow`);

8
web-console/src/druid-models/workbench-query/workbench-query-part.ts
View File

@ -22,7 +22,7 @@ import Hjson from 'hjson';
import * as JSONBig from 'json-bigint-native';
import type { ColumnMetadata } from '../../utils';
import { compact, filterMap, generate8HexId, sqlTypeFromDruid } from '../../utils';
import { compact, filterMap, generate8HexId } from '../../utils';
import type { LastExecution } from '../execution/execution';
import { validateLastExecution } from '../execution/execution';
import { fitExternalConfigPattern } from '../external-config/external-config';
@ -196,9 +196,9 @@ export class WorkbenchQueryPart {
const { queryName, parsedQuery } = this;
if (queryName && parsedQuery) {
try {
return fitExternalConfigPattern(parsedQuery).signature.map(({ name, type }) => ({
COLUMN_NAME: name,
DATA_TYPE: sqlTypeFromDruid(type),
return fitExternalConfigPattern(parsedQuery).signature.map(columnDeclaration => ({
COLUMN_NAME: columnDeclaration.getColumnName(),
DATA_TYPE: columnDeclaration.columnType.getEffectiveType(),
TABLE_NAME: queryName,
TABLE_SCHEMA: 'druid',
}));

159
web-console/src/druid-models/workbench-query/workbench-query.spec.ts
View File

@ -88,11 +88,10 @@ describe('WorkbenchQuery', () => {
SELECT *, VAR
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/data/kttm/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}'
)
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY ALL TIME
CLUSTERED BY browser, session
`;
@ -138,10 +137,9 @@ describe('WorkbenchQuery', () => {
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"}]'
'{"type":"json"}'
)
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY HOUR
CLUSTERED BY browser, session
`);
@ -153,10 +151,9 @@ describe('WorkbenchQuery', () => {
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"}]'
'{"type":"json"}'
)
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
`);
});
});
@ -447,12 +444,11 @@ describe('WorkbenchQuery', () => {
TIME_PARSE(pickup_datetime) AS __time,
*
FROM TABLE(
EXTERN(
'{"type": "local", ...}',
'{"type":"csv", ...}',
'[{ "name": "cab_type", "type": "string" }, ...]'
)
EXTERN(
'{"type": "local", ...}',
'{"type":"csv", ...}'
)
) EXTEND (cab_type, VARCHAR)
CLUSTERED BY trip_id
`;
@ -483,12 +479,11 @@ describe('WorkbenchQuery', () => {
TIME_PARSE(pickup_datetime) AS __time,
*
FROM TABLE(
EXTERN(
'{"type": "local", ...}',
'{"type":"csv", ...}',
'[{ "name": "cab_type", "type": "string" }, ...]'
)
EXTERN(
'{"type": "local", ...}',
'{"type":"csv", ...}'
)
) EXTEND (cab_type, VARCHAR)
CLUSTERED BY trip_id
`;
@ -567,52 +562,82 @@ describe('WorkbenchQuery', () => {
});
describe('#materializeHelpers', () => {
expect(
WorkbenchQuery.blank()
.changeQueryParts([
new WorkbenchQueryPart({
id: 'aaa',
queryName: 'kttm_data',
queryString: sane`
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
)
)
`,
}),
new WorkbenchQueryPart({
id: 'bbb',
queryName: 'country_lookup',
queryString: sane`
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}',
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
)
)
`,
}),
new WorkbenchQueryPart({
id: 'ccc',
queryName: 'x',
queryString: sane`
SELECT
os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",
COUNT(DISTINCT session) AS "unique_sessions"
FROM kttm_data
LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country
GROUP BY 1, 2
ORDER BY 3 DESC
LIMIT 10
`,
}),
])
.materializeHelpers(),
);
it('works', () => {
expect(
WorkbenchQuery.blank()
.changeQueryParts([
new WorkbenchQueryPart({
id: 'aaa',
queryName: 'kttm_data',
queryString: sane`
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
`,
}),
new WorkbenchQueryPart({
id: 'bbb',
queryName: 'country_lookup',
queryString: sane`
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}'
)
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
`,
}),
new WorkbenchQueryPart({
id: 'ccc',
queryName: 'x',
queryString: sane`
SELECT
os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",
COUNT(DISTINCT session) AS "unique_sessions"
FROM kttm_data
LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country
GROUP BY 1, 2
ORDER BY 3 DESC
LIMIT 10
`,
}),
])
.materializeHelpers()
.getQueryString(),
).toEqual(sane`
WITH
"kttm_data" AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
),
"country_lookup" AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}'
)
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
)
(
SELECT
os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",
COUNT(DISTINCT session) AS "unique_sessions"
FROM kttm_data
LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country
GROUP BY 1, 2
ORDER BY 3 DESC
LIMIT 10
)
`);
});
});
});

190
web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap Normal file
View File

@ -0,0 +1,190 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`spec conversion converts index_hadoop spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "newSource" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"s3","uris":["s3://path"]}',
'{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}'
)
) EXTEND ("event_ts" VARCHAR, "col1" VARCHAR, "col2" VARCHAR, "col3" VARCHAR, "col4" VARCHAR, "field1" DOUBLE, "field2" DOUBLE, "field3" DOUBLE, "field4" VARCHAR, "field5" VARCHAR, "field6" BIGINT, "field7" DOUBLE))
SELECT
TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time,
"col1",
"col2",
"col3",
"col4",
SUM("field1") AS "metric1",
MAX("field2") AS "metric2",
MIN("field3") AS "metric3",
APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4",
APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5",
SUM("field6") AS "metric6",
SUM("field7") AS "metric7"
FROM "source"
WHERE "col2" = 'xxx'
GROUP BY 1, 2, 3, 4, 5
PARTITIONED BY HOUR
`;
exports[`spec conversion converts index_parallel spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia_rollup" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "comment" VARCHAR, "isNew" VARCHAR, "isMinor" VARCHAR, "isAnonymous" VARCHAR, "user" VARCHAR, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "added" BIGINT, "commentLength" BIGINT, "delta" BIGINT, "deltaBucket" BIGINT, "deleted" BIGINT, "page" VARCHAR))
SELECT
TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"comment",
"isNew",
"isMinor",
"isAnonymous",
"user",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
COUNT(*) AS "count",
SUM("added") AS "sum_added",
SUM("commentLength") AS "sum_commentLength",
SUM("delta") AS "sum_delta",
SUM("deltaBucket") AS "sum_deltaBucket",
SUM("deleted") AS "sum_deleted",
APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta"
FROM "source"
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
PARTITIONED BY HOUR
`;
exports[`spec conversion converts index_parallel spec (without rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "event" TYPE('COMPLEX<json>')))
SELECT
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
"event"
FROM "source"
WHERE NOT ("channel" = 'xxx')
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
exports[`spec conversion converts with issue when there is a __time transform 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
exports[`spec conversion converts with issue when there is a dimension transform and strange filter 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;

201
web-console/src/helpers/spec-conversion.spec.ts
View File

@ -16,10 +16,13 @@
* limitations under the License.
*/
import { sane } from 'druid-query-toolkit';
import { convertSpecToSql } from './spec-conversion';
expect.addSnapshotSerializer({
test: val => typeof val === 'string',
print: String,
});
describe('spec conversion', () => {
it('converts index_parallel spec (without rollup)', () => {
const converted = convertSpecToSql({
@ -96,6 +99,7 @@ describe('spec conversion', () => {
'metroCode',
'countryIsoCode',
'regionName',
{ name: 'event', type: 'json' },
],
},
},
@ -116,46 +120,7 @@ describe('spec conversion', () => {
},
});
expect(converted.queryString).toEqual(sane`
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE NOT ("channel" = 'xxx')
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false,
@ -259,45 +224,7 @@ describe('spec conversion', () => {
},
});
expect(converted.queryString).toEqual(sane`
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia_rollup" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"comment","type":"string"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"},{"name":"added","type":"long"},{"name":"commentLength","type":"long"},{"name":"delta","type":"long"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"page","type":"string"}]'
)
))
SELECT
TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"comment",
"isNew",
"isMinor",
"isAnonymous",
"user",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
COUNT(*) AS "count",
SUM("added") AS "sum_added",
SUM("commentLength") AS "sum_commentLength",
SUM("delta") AS "sum_delta",
SUM("deltaBucket") AS "sum_deltaBucket",
SUM("deleted") AS "sum_deleted",
APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta"
FROM "source"
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
PARTITIONED BY HOUR
`);
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false,
@ -421,34 +348,7 @@ describe('spec conversion', () => {
},
});
expect(converted.queryString).toEqual(sane`
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "newSource" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"s3","uris":["s3://path"]}',
'{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}',
'[{"name":"event_ts","type":"string"},{"name":"col1","type":"string"},{"name":"col2","type":"string"},{"name":"col3","type":"string"},{"name":"col4","type":"string"},{"name":"field1","type":"double"},{"name":"field2","type":"double"},{"name":"field3","type":"double"},{"name":"field4","type":"string"},{"name":"field5","type":"string"},{"name":"field6","type":"long"},{"name":"field7","type":"double"}]'
)
))
SELECT
TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time,
"col1",
"col2",
"col3",
"col4",
SUM("field1") AS "metric1",
MAX("field2") AS "metric2",
MIN("field3") AS "metric3",
APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4",
APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5",
SUM("field6") AS "metric6",
SUM("field7") AS "metric7"
FROM "source"
WHERE "col2" = 'xxx'
GROUP BY 1, 2, 3, 4, 5
PARTITIONED BY HOUR
`);
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false,
@ -541,46 +441,7 @@ describe('spec conversion', () => {
},
});
expect(converted.queryString).toEqual(sane`
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
expect(converted.queryString).toMatchSnapshot();
});
it('converts with issue when there is a dimension transform and strange filter', () => {
@ -671,46 +532,6 @@ describe('spec conversion', () => {
},
});
expect(converted.queryString).toEqual(sane`
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
expect(converted.queryString).toMatchSnapshot();
});
});

65
web-console/src/helpers/spec-conversion.ts
View File

@ -16,7 +16,15 @@
* limitations under the License.
*/
import { C, L, RefName, SqlExpression, T } from 'druid-query-toolkit';
import {
C,
L,
RefName,
SqlColumnDeclaration,
SqlExpression,
SqlType,
T,
} from 'druid-query-toolkit';
import * as JSONBig from 'json-bigint-native';
import type {
@ -87,20 +95,22 @@ export function convertSpecToSql(spec: any): QueryWithContext {
}
dimensions = dimensions.map(inflateDimensionSpec);
let columns = dimensions.map((d: DimensionSpec) => ({
name: d.name,
type: d.type,
}));
let columnDeclarations: SqlColumnDeclaration[] = dimensions.map((d: DimensionSpec) =>
SqlColumnDeclaration.create(
d.name,
SqlType.fromNativeType(dimensionSpecTypeToNativeDataType(d.type)),
),
);
const metricsSpec = deepGet(spec, 'spec.dataSchema.metricsSpec');
if (Array.isArray(metricsSpec)) {
columns = columns.concat(
columnDeclarations = columnDeclarations.concat(
filterMap(metricsSpec, metricSpec =>
metricSpec.fieldName
? {
name: metricSpec.fieldName,
type: metricSpecTypeToDataType(metricSpec.type),
}
? SqlColumnDeclaration.create(
metricSpec.fieldName,
SqlType.fromNativeType(metricSpecTypeToNativeDataInputType(metricSpec.type)),
)
: undefined,
),
);
@ -119,42 +129,46 @@ export function convertSpecToSql(spec: any): QueryWithContext {
if (timeTransform) {
timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
} else {
let timestampColumnType: SqlType;
switch (format) {
case 'auto':
columns.unshift({ name: timestampColumnName, type: 'string' });
timestampColumnType = SqlType.VARCHAR;
timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE TIME_PARSE(${timestampColumn}) END`;
break;
case 'iso':
columns.unshift({ name: timestampColumnName, type: 'string' });
timestampColumnType = SqlType.VARCHAR;
timeExpression = `TIME_PARSE(${timestampColumn})`;
break;
case 'posix':
columns.unshift({ name: timestampColumnName, type: 'long' });
timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`;
break;
case 'millis':
columns.unshift({ name: timestampColumnName, type: 'long' });
timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`;
break;
case 'micro':
columns.unshift({ name: timestampColumnName, type: 'long' });
timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`;
break;
case 'nano':
columns.unshift({ name: timestampColumnName, type: 'long' });
timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`;
break;
default:
columns.unshift({ name: timestampColumnName, type: 'string' });
timestampColumnType = SqlType.VARCHAR;
timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`;
break;
}
columnDeclarations.unshift(
SqlColumnDeclaration.create(timestampColumnName, timestampColumnType),
);
}
if (timestampSpec.missingValue) {
@ -238,10 +252,9 @@ export function convertSpecToSql(spec: any): QueryWithContext {
const inputFormat = deepGet(spec, 'spec.ioConfig.inputFormat');
if (!inputFormat) throw new Error(`spec.ioConfig.inputFormat is not defined`);
lines.push(
` ${L(JSONBig.stringify(inputFormat))},`,
` ${L(JSONBig.stringify(columns))}`,
` ${L(JSONBig.stringify(inputFormat))}`,
` )`,
`))`,
`) EXTEND (${columnDeclarations.join(', ')}))`,
);
}
@ -362,7 +375,17 @@ const QUERY_GRANULARITY_MAP: Record<string, string> = {
year: `TIME_FLOOR(?, 'P1Y')`,
};
function metricSpecTypeToDataType(metricSpecType: string): string {
function dimensionSpecTypeToNativeDataType(dimensionSpecType: string): string {
switch (dimensionSpecType) {
case 'json':
return 'COMPLEX<json>';
default:
return dimensionSpecType;
}
}
function metricSpecTypeToNativeDataInputType(metricSpecType: string): string {
const m = /^(long|float|double|string)/.exec(String(metricSpecType));
if (m) return m[1];

18
web-console/src/utils/types.ts
View File

@ -25,24 +25,6 @@ function getEffectiveColumnType(column: Column): string | undefined {
return column.nativeType || column.sqlType;
}
export function sqlTypeFromDruid(druidType: string): string {
druidType = druidType.toLowerCase();
switch (druidType) {
case 'string':
return 'VARCHAR';
case 'long':
return 'BIGINT';
case 'float':
case 'double':
return druidType.toUpperCase();
default:
return 'COMPLEX';
}
}
export function columnToIcon(column: Column): IconName | undefined {
const effectiveType = getEffectiveColumnType(column);
return effectiveType ? dataTypeToIcon(effectiveType) : undefined;

32
web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
View File

@ -42,23 +42,25 @@ function breakdownExpression(expression: SqlExpression): Breakdown {
expression = expression.getUnderlyingExpression();
let nativeType: string | undefined;
if (expression instanceof SqlFunction && expression.getEffectiveFunctionName() === 'CAST') {
const asType = String(expression.getArgAsString(1)).toUpperCase();
switch (asType) {
case 'VARCHAR':
nativeType = 'string';
expression = expression.getArg(0)!;
break;
if (expression instanceof SqlFunction) {
const asType = expression.getCastType();
if (asType) {
switch (asType.value.toUpperCase()) {
case 'VARCHAR':
nativeType = 'string';
expression = expression.getArg(0)!;
break;
case 'BIGINT':
nativeType = 'long';
expression = expression.getArg(0)!;
break;
case 'BIGINT':
nativeType = 'long';
expression = expression.getArg(0)!;
break;
case 'DOUBLE':
nativeType = 'double';
expression = expression.getArg(0)!;
break;
case 'DOUBLE':
nativeType = 'double';
expression = expression.getArg(0)!;
break;
}
}
}

49
web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
View File

@ -451,11 +451,13 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
}, [previewResultState]);
const unusedColumns = ingestQueryPattern
? ingestQueryPattern.mainExternalConfig.signature.filter(
({ name }) =>
!ingestQueryPattern.dimensions.some(d => d.containsColumnName(name)) &&
!ingestQueryPattern.metrics?.some(m => m.containsColumnName(name)),
)
? ingestQueryPattern.mainExternalConfig.signature.filter(columnDeclaration => {
const columnName = columnDeclaration.getColumnName();
return (
!ingestQueryPattern.dimensions.some(d => d.containsColumnName(columnName)) &&
!ingestQueryPattern.metrics?.some(m => m.containsColumnName(columnName))
);
})
: [];
const timeColumn =
@ -699,23 +701,26 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
)}
<MenuDivider />
{unusedColumns.length ? (
unusedColumns.map((column, i) => (
<MenuItem
key={i}
icon={dataTypeToIcon(column.type)}
text={column.name}
onClick={() => {
handleQueryAction(q =>
q.addSelect(
C(column.name),
ingestQueryPattern.metrics
? { insertIndex: 'last-grouping', addToGroupBy: 'end' }
: {},
),
);
}}
/>
))
unusedColumns.map((columnDeclaration, i) => {
const columnName = columnDeclaration.getColumnName();
return (
<MenuItem
key={i}
icon={dataTypeToIcon(columnDeclaration.columnType.getNativeType())}
text={columnName}
onClick={() => {
handleQueryAction(q =>
q.addSelect(
C(columnName),
ingestQueryPattern.metrics
? { insertIndex: 'last-grouping', addToGroupBy: 'end' }
: {},
),
);
}}
/>
);
})
) : (
<MenuItem icon={IconNames.BLANK} text="No column suggestions" disabled />
)}

35
web-console/src/views/workbench-view/demo-queries.ts
View File

@ -54,10 +54,9 @@ SELECT *
FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"json"}'
)
)
) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)
PARTITIONED BY ALL TIME
`,
),
@ -85,10 +84,9 @@ WITH kttm_data AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"json"}'
)
))
) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR))
SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -132,18 +130,16 @@ kttm_data AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"json"}'
)
)),
) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)),
country_lookup AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}',
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
'{"type":"tsv","findColumnsFromHeader":true}'
)
))
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -190,10 +186,9 @@ country_lookup AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}',
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
'{"type":"tsv","findColumnsFromHeader":true}'
)
))
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -261,18 +256,16 @@ kttm_data AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
'{"type":"json"}'
)
)),
) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)),
country_lookup AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}',
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
'{"type":"tsv","findColumnsFromHeader":true}'
)
))
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT
os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",

1004
web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap
View File

File diff suppressed because it is too large Load Diff

4
web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
View File

@ -16,12 +16,12 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `
</Memo(ExternalLink)>
:
</React.Fragment>
Too many warnings of type CannotParseExternalData generated (max = 10)
Too many warnings of type CannotParseExternalData generated (max = 2)
</p>
<div>
Failed task ID:
<Memo(ClickToCopy)
text="query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a"
text="query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0"
/>
<React.Fragment>
(on host:

21
web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap
View File

@ -158,7 +158,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
Array [
Object {
"definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_0",
"id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0",
"input": Array [
Object {
"inputFormat": Object {
@ -186,6 +186,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"type": "external",
},
],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1,
"processor": Object {
"query": Object {
@ -198,10 +199,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"finalize": false,
"finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9",
"sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all",
},
"dataSource": Object {
@ -237,6 +239,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"columns": Array [
Object {
"columnName": "__boost",
"order": "ASCENDING",
},
],
},
@ -258,23 +261,24 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
},
],
},
"duration": 4887,
"duration": 24236,
"partitionCount": 1,
"phase": "FINISHED",
"sort": true,
"stageNumber": 0,
"startTime": "2022-10-31T16:11:15.380Z",
"startTime": "2023-03-27T22:17:02.792Z",
"workerCount": 1,
},
Object {
"definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_1",
"id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1",
"input": Array [
Object {
"stage": 0,
"type": "stage",
},
],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1,
"processor": Object {
"columnMappings": Array [
@ -302,6 +306,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
},
],
"includeAllDimensions": false,
"useSchemaDiscovery": false,
},
"granularitySpec": Object {
"intervals": Array [
@ -333,11 +338,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
},
"signature": Array [],
},
"duration": 1908,
"duration": 4276,
"partitionCount": 1,
"phase": "FINISHED",
"stageNumber": 1,
"startTime": "2022-10-31T16:11:20.264Z",
"startTime": "2023-03-27T22:17:26.978Z",
"workerCount": 1,
},
]

2
web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap
View File

@ -8,7 +8,7 @@ exports[`IngestSuccessPane matches snapshot 1`] = `
465,346 rows inserted into "kttm_simple".
</p>
<p>
Insert query took 0:00:09.
Insert query took 0:00:28.
<span
className="action"
onClick={[Function]}

24
web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
View File

@ -19,11 +19,11 @@
import { Button, Callout, FormGroup, Icon, Intent, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { SqlExpression } from 'druid-query-toolkit';
import { C } from 'druid-query-toolkit';
import { C, SqlColumnDeclaration, SqlType } from 'druid-query-toolkit';
import React, { useState } from 'react';
import { AutoForm, CenterMessage, LearnMore, Loader } from '../../../components';
import type { InputFormat, InputSource, SignatureColumn } from '../../../druid-models';
import type { InputFormat, InputSource } from '../../../druid-models';
import {
guessColumnTypeFromHeaderAndRows,
guessIsArrayFromHeaderAndRows,
@ -51,7 +51,7 @@ const noop = () => {};
export interface InputFormatAndMore {
inputFormat: InputFormat;
signature: SignatureColumn[];
signature: SqlColumnDeclaration[];
isArrays: boolean[];
timeExpression: SqlExpression | undefined;
}
@ -121,7 +121,7 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
let possibleTimeExpression: PossibleTimeExpression | undefined;
if (previewData) {
possibleTimeExpression = filterMap(previewData.header, column => {
const values = previewData.rows.map(row => row.input[column]);
const values = filterMap(previewData.rows, row => row.input?.[column]);
const possibleDruidFormat = possibleDruidFormatForValues(values);
if (!possibleDruidFormat) return;
@ -139,14 +139,18 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
previewData && AutoForm.isValidModel(inputFormat, INPUT_FORMAT_FIELDS)
? {
inputFormat,
signature: previewData.header.map(name => ({
name,
type: guessColumnTypeFromHeaderAndRows(
previewData,
signature: previewData.header.map(name =>
SqlColumnDeclaration.create(
name,
inputFormatOutputsNumericStrings(inputFormat),
SqlType.fromNativeType(
guessColumnTypeFromHeaderAndRows(
previewData,
name,
inputFormatOutputsNumericStrings(inputFormat),
),
),
),
})),
),
isArrays: previewData.header.map(name =>
guessIsArrayFromHeaderAndRows(previewData, name),
),

3
web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
View File

@ -29,6 +29,7 @@ import {
import { IconNames } from '@blueprintjs/icons';
import classNames from 'classnames';
import type { QueryResult } from 'druid-query-toolkit';
import { SqlColumnDeclaration } from 'druid-query-toolkit';
import React, { useEffect, useState } from 'react';
import { AutoForm, ExternalLink } from '../../../components';
@ -138,7 +139,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS
listDelimiter: BOGUS_LIST_DELIMITER,
columns: ['raw'],
},
signature: [{ name: 'raw', type: 'string' }],
signature: [SqlColumnDeclaration.create('raw', 'VARCHAR')],
});
const result = extractResult(

5
web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
View File

@ -165,10 +165,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result
// Casts
if (selectExpression) {
const underlyingExpression = selectExpression.getUnderlyingExpression();
if (
underlyingExpression instanceof SqlFunction &&
underlyingExpression.getEffectiveFunctionName() === 'CAST'
) {
if (underlyingExpression instanceof SqlFunction && underlyingExpression.getCastType()) {
menuItems.push(
<MenuItem
key="uncast"

20
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -54,7 +54,7 @@ import {
getUseCache,
summarizeIndexSpec,
} from '../../../druid-models';
import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils';
import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
import './run-panel.scss';
@ -107,6 +107,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const maxParseExceptions = getMaxParseExceptions(queryContext);
const finalizeAggregations = getFinalizeAggregations(queryContext);
const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast';
const durableShuffleStorage = getDurableShuffleStorage(queryContext);
const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext);
@ -302,6 +303,23 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v))
}
/>
<MenuItem
icon={IconNames.INNER_JOIN}
text="Join algorithm"
label={sqlJoinAlgorithm}
>
{['broadcast', 'sortMerge'].map(o => (
<MenuItem
key={o}
icon={tickIcon(sqlJoinAlgorithm === o)}
text={o}
shouldDismissPopover={false}
onClick={() =>
changeQueryContext(deepSet(queryContext, 'sqlJoinAlgorithm', o))
}
/>
))}
</MenuItem>
<MenuItem
icon={IconNames.TH_DERIVED}
text="Edit index spec"