Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: use EXTEND syntax (#13985) 

* use EXTEND syntax

* update licenses

* update demo queries

* updated snapshots

* add join algorithm selector

* dismiss
This commit is contained in:
Vadim Ogievetsky 2023-03-29 16:19:49 -07:00 committed by GitHub
parent f715887172
commit abb7133153
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 1388 additions and 1275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
licenses.yaml
View File

@ -5080,7 +5080,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Sebastian McKenzie copyright: Sebastian McKenzie
version: 7.5.5 version: 7.12.11
license_file_path: licenses/bin/@babel-code-frame.MIT license_file_path: licenses/bin/@babel-code-frame.MIT
--- ---
@ -5099,8 +5099,8 @@ name: "@babel/helper-validator-identifier"
license_category: binary license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Sebastian McKenzie and other contributors copyright: The Babel Team
version: 7.14.0 version: 7.19.1
license_file_path: licenses/bin/@babel-helper-validator-identifier.MIT license_file_path: licenses/bin/@babel-helper-validator-identifier.MIT
--- ---
@ -5109,8 +5109,8 @@ name: "@babel/highlight"
license_category: binary license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: suchipi copyright: The Babel Team
version: 7.5.0 version: 7.18.6
license_file_path: licenses/bin/@babel-highlight.MIT license_file_path: licenses/bin/@babel-highlight.MIT
--- ---
@ -5120,7 +5120,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: The Babel Team copyright: The Babel Team
version: 7.14.8 version: 7.21.0
license_file_path: licenses/bin/@babel-runtime.MIT license_file_path: licenses/bin/@babel-runtime.MIT
--- ---
@ -5140,7 +5140,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Palantir Technologies copyright: Palantir Technologies
version: 4.0.1 version: 4.1.19
--- ---
@ -5149,7 +5149,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Palantir Technologies copyright: Palantir Technologies
version: 4.0.3 version: 4.17.6
--- ---
@ -5158,7 +5158,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Palantir Technologies copyright: Palantir Technologies
version: 4.0.3 version: 4.4.25
--- ---
@ -5167,7 +5167,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Palantir Technologies copyright: Palantir Technologies
version: 4.1.1 version: 4.14.3
--- ---
@ -5176,7 +5176,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Palantir Technologies copyright: Palantir Technologies
version: 1.0.3 version: 1.13.10
--- ---
@ -5285,7 +5285,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Juggle copyright: Juggle
version: 3.3.1 version: 3.4.0
--- ---
@ -5294,7 +5294,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Federico Zivolo copyright: Federico Zivolo
version: 2.11.4 version: 2.11.6
license_file_path: licenses/bin/@popperjs-core.MIT license_file_path: licenses/bin/@popperjs-core.MIT
--- ---
@ -5319,6 +5319,36 @@ license_file_path: licenses/bin/@types-parse-json.MIT
--- ---
name: "@types/prop-types"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 15.7.3
license_file_path: licenses/bin/@types-prop-types.MIT
---
name: "@types/react"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 16.14.35
license_file_path: licenses/bin/@types-react.MIT
---
name: "@types/scheduler"
license_category: binary
module: web-console
license_name: MIT License
copyright: Microsoft Corporation.
version: 0.16.1
license_file_path: licenses/bin/@types-scheduler.MIT
---
name: "ace-builds" name: "ace-builds"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5379,6 +5409,16 @@ license_file_path: licenses/bin/babel-plugin-syntax-jsx.MIT
--- ---
name: "call-bind"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.2
license_file_path: licenses/bin/call-bind.MIT
---
name: "callsites" name: "callsites"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5434,7 +5474,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Jed Watson copyright: Jed Watson
version: 2.2.6 version: 2.3.2
license_file_path: licenses/bin/classnames.MIT license_file_path: licenses/bin/classnames.MIT
--- ---
@ -5634,7 +5674,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Jordan Harband copyright: Jordan Harband
version: 1.1.3 version: 1.2.0
license_file_path: licenses/bin/define-properties.MIT license_file_path: licenses/bin/define-properties.MIT
--- ---
@ -5693,7 +5733,7 @@ license_category: binary
module: web-console module: web-console
license_name: Apache License version 2.0 license_name: Apache License version 2.0
copyright: Imply Data copyright: Imply Data
version: 0.17.5 version: 0.18.2
--- ---
@ -5727,16 +5767,6 @@ license_file_path: licenses/bin/escape-string-regexp.MIT
--- ---
name: "esutils"
license_category: binary
module: web-console
license_name: BSD-2-Clause License
copyright: Yusuke Suzuki
version: 2.0.3
license_file_path: licenses/bin/esutils.BSD2
---
name: "file-saver" name: "file-saver"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5787,6 +5817,26 @@ license_file_path: licenses/bin/function-bind.MIT
--- ---
name: "functions-have-names"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.2.3
license_file_path: licenses/bin/functions-have-names.MIT
---
name: "get-intrinsic"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.2.0
license_file_path: licenses/bin/get-intrinsic.MIT
---
name: "gud" name: "gud"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5817,6 +5867,36 @@ license_file_path: licenses/bin/has-own-prop.MIT
--- ---
name: "has-property-descriptors"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/has-property-descriptors.MIT
---
name: "has-symbols"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.3
license_file_path: licenses/bin/has-symbols.MIT
---
name: "has-tostringtag"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/has-tostringtag.MIT
---
name: "has" name: "has"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5907,6 +5987,16 @@ license_file_path: licenses/bin/is-arrayish.MIT
--- ---
name: "is-core-module"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 2.10.0
license_file_path: licenses/bin/is-core-module.MIT
---
name: "is-date-object" name: "is-date-object"
license_category: binary license_category: binary
module: web-console module: web-console
@ -5922,7 +6012,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Jordan Harband copyright: Jordan Harband
version: 1.0.4 version: 1.1.4
license_file_path: licenses/bin/is-regex.MIT license_file_path: licenses/bin/is-regex.MIT
--- ---
@ -6341,7 +6431,7 @@ license_category: binary
module: web-console module: web-console
license_name: BSD-3-Clause License license_name: BSD-3-Clause License
copyright: React Community copyright: React Community
version: 4.4.2 version: 4.4.5
license_file_path: licenses/bin/react-transition-group.BSD3 license_file_path: licenses/bin/react-transition-group.BSD3
--- ---
@ -6361,7 +6451,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Ben Newman copyright: Ben Newman
version: 0.13.7 version: 0.13.11
license_file_path: licenses/bin/regenerator-runtime.MIT license_file_path: licenses/bin/regenerator-runtime.MIT
--- ---
@ -6371,7 +6461,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: Jordan Harband copyright: Jordan Harband
version: 1.2.0 version: 1.4.3
license_file_path: licenses/bin/regexp.prototype.flags.MIT license_file_path: licenses/bin/regexp.prototype.flags.MIT
--- ---
@ -6401,7 +6491,7 @@ license_category: binary
module: web-console module: web-console
license_name: MIT License license_name: MIT License
copyright: James Halliday copyright: James Halliday
version: 1.12.0 version: 1.22.1
license_file_path: licenses/bin/resolve.MIT license_file_path: licenses/bin/resolve.MIT
--- ---
@ -6466,6 +6556,16 @@ license_file_path: licenses/bin/supports-color.MIT
--- ---
name: "supports-preserve-symlinks-flag"
license_category: binary
module: web-console
license_name: MIT License
copyright: Jordan Harband
version: 1.0.0
license_file_path: licenses/bin/supports-preserve-symlinks-flag.MIT
---
name: "tiny-invariant" name: "tiny-invariant"
license_category: binary license_category: binary
module: web-console module: web-console

21
licenses/bin/@types-prop-types.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/@types-react.MIT Executable file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/@types-scheduler.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

21
licenses/bin/call-bind.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

2
licenses/bin/classnames.MIT
View File

@ -1,6 +1,6 @@
The MIT License (MIT) The MIT License (MIT)
Copyright (c) 2017 Jed Watson Copyright (c) 2018 Jed Watson
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

21
licenses/bin/functions-have-names.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/get-intrinsic.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-property-descriptors.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-symbols.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2016 Jordan Harband
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

21
licenses/bin/has-tostringtag.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

20
licenses/bin/is-core-module.MIT Normal file
View File

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Dave Justice
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

3
licenses/bin/regexp.prototype.flags.MIT
View File

@ -18,4 +18,5 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. THE SOFTWARE.

21
licenses/bin/supports-preserve-symlinks-flag.MIT Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Inspect JS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

5
web-console/e2e-tests/multi-stage-query.spec.ts
View File

@ -52,10 +52,9 @@ FROM TABLE(
'{"type":"local","filter":"wikiticker-2015-09-12-sampled.json.gz","baseDir":${JSON.stringify( '{"type":"local","filter":"wikiticker-2015-09-12-sampled.json.gz","baseDir":${JSON.stringify(
DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
)}}', )}}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"channel","type":"string"}]'
) )
)) ) EXTEND (channel VARCHAR))
SELECT SELECT
channel, channel,
CAST(COUNT(*) AS VARCHAR) AS "CountString" CAST(COUNT(*) AS VARCHAR) AS "CountString"

14
web-console/package-lock.json generated
View File

@ -22,7 +22,7 @@
"d3-axis": "^2.1.0", "d3-axis": "^2.1.0",
"d3-scale": "^3.3.0", "d3-scale": "^3.3.0",
"d3-selection": "^2.0.0", "d3-selection": "^2.0.0",
"druid-query-toolkit": "^0.17.5", "druid-query-toolkit": "^0.18.2",
"file-saver": "^2.0.2", "file-saver": "^2.0.2",
"follow-redirects": "^1.14.7", "follow-redirects": "^1.14.7",
"fontsource-open-sans": "^3.0.9", "fontsource-open-sans": "^3.0.9",
@ -8211,9 +8211,9 @@
} }
}, },
"node_modules/druid-query-toolkit": { "node_modules/druid-query-toolkit": {
"version": "0.17.5", "version": "0.18.2",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz", "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz",
"integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==", "integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==",
"dependencies": { "dependencies": {
"tslib": "^2.3.1" "tslib": "^2.3.1"
}, },
@ -32625,9 +32625,9 @@
} }
}, },
"druid-query-toolkit": { "druid-query-toolkit": {
"version": "0.17.5", "version": "0.18.2",
"resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz", "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz",
"integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==", "integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==",
"requires": { "requires": {
"tslib": "^2.3.1" "tslib": "^2.3.1"
} }

5
web-console/package.json
View File

@ -17,9 +17,6 @@
"testMatch": [ "testMatch": [
"**/?(*.)+(spec).ts?(x)" "**/?(*.)+(spec).ts?(x)"
], ],
"snapshotSerializers": [
"enzyme-to-json/serializer"
],
"setupFilesAfterEnv": [ "setupFilesAfterEnv": [
"<rootDir>src/setup-tests.ts" "<rootDir>src/setup-tests.ts"
] ]
@ -79,7 +76,7 @@
"d3-axis": "^2.1.0", "d3-axis": "^2.1.0",
"d3-scale": "^3.3.0", "d3-scale": "^3.3.0",
"d3-selection": "^2.0.0", "d3-selection": "^2.0.0",
"druid-query-toolkit": "^0.17.5", "druid-query-toolkit": "^0.18.2",
"file-saver": "^2.0.2", "file-saver": "^2.0.2",
"follow-redirects": "^1.14.7", "follow-redirects": "^1.14.7",
"fontsource-open-sans": "^3.0.9", "fontsource-open-sans": "^3.0.9",

11
web-console/script/create-sql-docs.js
View File

@ -23,9 +23,16 @@ const snarkdown = require('snarkdown');
const writefile = 'lib/sql-docs.js'; const writefile = 'lib/sql-docs.js';
const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162; const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 164;
const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14; const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
const initialFunctionDocs = {
TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an external.')]],
EXTERN: [
['inputSource, inputFormat, rowSignature?', convertMarkdownToHtml('Reads external data')],
],
};
function hasHtmlTags(str) { function hasHtmlTags(str) {
return /<(a|br|span|div|p|code)\/?>/.test(str); return /<(a|br|span|div|p|code)\/?>/.test(str);
} }
@ -67,7 +74,7 @@ const readDoc = async () => {
const lines = data.split('\n'); const lines = data.split('\n');
const functionDocs = {}; const functionDocs = initialFunctionDocs;
const dataTypeDocs = {}; const dataTypeDocs = {};
for (let line of lines) { for (let line of lines) {
const functionMatch = line.match(/^\|\s*`(\w+)\(([^|]*)\)`\s*\|([^|]+)\|(?:([^|]+)\|)?$/); const functionMatch = line.match(/^\|\s*`(\w+)\(([^|]*)\)`\s*\|([^|]+)\|(?:([^|]+)\|)?$/);

4
web-console/src/bootstrap/ace.scss
View File

@ -35,6 +35,10 @@
text-decoration: underline; text-decoration: underline;
text-decoration-style: wavy; text-decoration-style: wavy;
} }
.ace_storage.ace_type {
color: #27c923;
}
// END: Custom code styles // END: Custom code styles
&.no-background { &.no-background {

88
web-console/src/druid-models/execution/execution-ingest-complete.mock.ts
View File

@ -19,26 +19,33 @@
import { Execution } from './execution'; import { Execution } from './execution';
/* /*
For query: ===== Query =====
REPLACE INTO "kttm_simple" OVERWRITE ALL REPLACE INTO "kttm_simple" OVERWRITE ALL
SELECT TIME_PARSE("timestamp") AS "__time", agent_type SELECT
TIME_PARSE("timestamp") AS "__time",
"agent_type"
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY ALL TIME PARTITIONED BY ALL TIME
===== Context =====
{
"maxNumTasks": 2
}
*/ */
export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{ {
task: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', task: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
payload: { payload: {
type: 'query_controller', type: 'query_controller',
id: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', id: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
spec: { spec: {
query: { query: {
queryType: 'scan', queryType: 'scan',
@ -78,10 +85,11 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
finalize: false, finalize: false,
finalizeAggregations: false, finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', maxNumTasks: 2,
queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
}, },
granularity: { type: 'all' }, granularity: { type: 'all' },
@ -100,22 +108,23 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 },
}, },
sqlQuery: sqlQuery:
'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL TIME', 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME',
sqlQueryContext: { sqlQueryContext: {
finalizeAggregations: false, finalizeAggregations: false,
maxParseExceptions: 0, maxParseExceptions: 0,
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '{"type":"all"}',
maxNumTasks: 2,
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
}, },
sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], sqlTypeNames: ['TIMESTAMP', 'VARCHAR'],
context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true },
groupId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', groupId: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
dataSource: 'kttm_simple', dataSource: 'kttm_simple',
resource: { resource: {
availabilityGroup: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', availabilityGroup: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
requiredCapacity: 1, requiredCapacity: 1,
}, },
}, },
@ -124,12 +133,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{ {
multiStageQuery: { multiStageQuery: {
type: 'multiStageQuery', type: 'multiStageQuery',
taskId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', taskId: 'query-b55f3432-7810-4529-80ed-780a926a6f03',
payload: { payload: {
status: { status: {
status: 'SUCCESS', status: 'SUCCESS',
startTime: '2022-10-31T16:11:13.160Z', startTime: '2023-03-27T22:17:02.401Z',
durationMs: 9012, durationMs: 28854,
pendingTasks: 0, pendingTasks: 0,
runningTasks: 2, runningTasks: 2,
}, },
@ -137,7 +146,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
{ {
stageNumber: 0, stageNumber: 0,
definition: { definition: {
id: 'b94caff0-f693-47ed-a242-ccf837550383_0', id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0',
input: [ input: [
{ {
type: 'external', type: 'external',
@ -184,11 +193,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
finalize: false, finalize: false,
finalizeAggregations: false, finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', maxNumTasks: 2,
queryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
scanSignature: scanSignature:
'[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '{"type":"all"}',
sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03',
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
}, },
granularity: { type: 'all' }, granularity: { type: 'all' },
@ -201,23 +211,24 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
], ],
shuffleSpec: { shuffleSpec: {
type: 'targetSize', type: 'targetSize',
clusterBy: { columns: [{ columnName: '__boost' }] }, clusterBy: { columns: [{ columnName: '__boost', order: 'ASCENDING' }] },
targetSize: 3000000, targetSize: 3000000,
}, },
maxWorkerCount: 1, maxWorkerCount: 1,
shuffleCheckHasMultipleValues: true, shuffleCheckHasMultipleValues: true,
maxInputBytesPerWorker: 10737418240,
}, },
phase: 'FINISHED', phase: 'FINISHED',
workerCount: 1, workerCount: 1,
partitionCount: 1, partitionCount: 1,
startTime: '2022-10-31T16:11:15.380Z', startTime: '2023-03-27T22:17:02.792Z',
duration: 4887, duration: 24236,
sort: true, sort: true,
}, },
{ {
stageNumber: 1, stageNumber: 1,
definition: { definition: {
id: 'b94caff0-f693-47ed-a242-ccf837550383_1', id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1',
input: [{ type: 'stage', stage: 0 }], input: [{ type: 'stage', stage: 0 }],
processor: { processor: {
type: 'segmentGenerator', type: 'segmentGenerator',
@ -235,6 +246,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
], ],
dimensionExclusions: ['__time'], dimensionExclusions: ['__time'],
includeAllDimensions: false, includeAllDimensions: false,
useSchemaDiscovery: false,
}, },
metricsSpec: [], metricsSpec: [],
granularitySpec: { granularitySpec: {
@ -257,32 +269,48 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport(
}, },
signature: [], signature: [],
maxWorkerCount: 1, maxWorkerCount: 1,
maxInputBytesPerWorker: 10737418240,
}, },
phase: 'FINISHED', phase: 'FINISHED',
workerCount: 1, workerCount: 1,
partitionCount: 1, partitionCount: 1,
startTime: '2022-10-31T16:11:20.264Z', startTime: '2023-03-27T22:17:26.978Z',
duration: 1908, duration: 4276,
}, },
], ],
counters: { counters: {
'0': { '0': {
'0': { '0': {
input0: { type: 'channel', rows: [465346], files: [1], totalFiles: [1] }, input0: {
type: 'channel',
rows: [465346],
bytes: [360464067],
files: [1],
totalFiles: [1],
},
output: { type: 'channel', rows: [465346], bytes: [25430674], frames: [4] }, output: { type: 'channel', rows: [465346], bytes: [25430674], frames: [4] },
shuffle: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] }, shuffle: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] },
sortProgress: { sortProgress: {
type: 'sortProgress', type: 'sortProgress',
totalMergingLevels: 3, totalMergingLevels: 3,
levelToTotalBatches: { '0': 2, '1': 1, '2': 1 }, levelToTotalBatches: { '0': 1, '1': 1, '2': 1 },
levelToMergedBatches: { '0': 2, '1': 1, '2': 1 }, levelToMergedBatches: { '0': 1, '1': 1, '2': 1 },
totalMergersForUltimateLevel: 1, totalMergersForUltimateLevel: 1,
progressDigest: 1.0, progressDigest: 1.0,
}, },
}, },
}, },
'1': { '1': {
'0': { input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] } }, '0': {
input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] },
segmentGenerationProgress: {
type: 'segmentGenerationProgress',
rowsProcessed: 465346,
rowsPersisted: 465346,
rowsMerged: 465346,
rowsPushed: 465346,
},
},
}, },
}, },
}, },

165
web-console/src/druid-models/execution/execution-ingest-error.mock.ts
View File

@ -21,30 +21,32 @@ import { Execution } from './execution';
/* /*
===== Query ===== ===== Query =====
REPLACE INTO "kttm_simple" OVERWRITE ALL REPLACE INTO "kttm-blank-lines" OVERWRITE ALL
SELECT TIME_PARSE("timestamp") AS "__time", agent_type SELECT
TIME_PARSE("timestamp") AS "__time",
"agent_type"
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY ALL PARTITIONED BY DAY
===== Context ===== ===== Context =====
{ {
"maxParseExceptions": 10 "maxParseExceptions": 2,
"maxNumTasks": 2
} }
*/ */
export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{ {
task: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', task: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
payload: { payload: {
type: 'query_controller', type: 'query_controller',
id: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', id: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
spec: { spec: {
query: { query: {
queryType: 'scan', queryType: 'scan',
@ -52,7 +54,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
type: 'external', type: 'external',
inputSource: { inputSource: {
type: 'http', type: 'http',
uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'], uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'],
}, },
inputFormat: { inputFormat: {
type: 'json', type: 'json',
@ -84,11 +86,12 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
finalize: false, finalize: false,
finalizeAggregations: false, finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 10, maxNumTasks: 2,
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', maxParseExceptions: 2,
queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '"DAY"',
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
}, },
granularity: { type: 'all' }, granularity: { type: 'all' },
@ -99,30 +102,31 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
], ],
destination: { destination: {
type: 'dataSource', type: 'dataSource',
dataSource: 'kttm_simple', dataSource: 'kttm-blank-lines',
segmentGranularity: { type: 'all' }, segmentGranularity: 'DAY',
replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'],
}, },
assignmentStrategy: 'max', assignmentStrategy: 'max',
tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 },
}, },
sqlQuery: sqlQuery:
'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL', 'REPLACE INTO "kttm-blank-lines" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY DAY',
sqlQueryContext: { sqlQueryContext: {
maxParseExceptions: 10, maxParseExceptions: 2,
finalizeAggregations: false, finalizeAggregations: false,
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '"DAY"',
maxNumTasks: 2,
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
}, },
sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], sqlTypeNames: ['TIMESTAMP', 'VARCHAR'],
context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true },
groupId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', groupId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
dataSource: 'kttm_simple', dataSource: 'kttm-blank-lines',
resource: { resource: {
availabilityGroup: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', availabilityGroup: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
requiredCapacity: 1, requiredCapacity: 1,
}, },
}, },
@ -131,85 +135,48 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{ {
multiStageQuery: { multiStageQuery: {
type: 'multiStageQuery', type: 'multiStageQuery',
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765',
payload: { payload: {
status: { status: {
status: 'FAILED', status: 'FAILED',
errorReport: { errorReport: {
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost', host: 'localhost',
error: { error: {
errorCode: 'TooManyWarnings', errorCode: 'TooManyWarnings',
maxWarnings: 10, maxWarnings: 2,
rootErrorCode: 'CannotParseExternalData', rootErrorCode: 'CannotParseExternalData',
errorMessage: errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)',
'Too many warnings of type CannotParseExternalData generated (max = 10)',
}, },
}, },
warnings: [ warnings: [
{ {
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost:8091', host: 'localhost:8101',
stageNumber: 0, stageNumber: 0,
error: { error: {
errorCode: 'CannotParseExternalData', errorCode: 'CannotParseExternalData',
errorMessage: errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)', 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)',
}, },
exceptionStackTrace: exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
}, },
{ {
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0',
host: 'localhost:8091', host: 'localhost:8101',
stageNumber: 0, stageNumber: 0,
error: { error: {
errorCode: 'CannotParseExternalData', errorCode: 'CannotParseExternalData',
errorMessage: errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)', 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)',
}, },
exceptionStackTrace: exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
},
{
taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0',
host: 'localhost:8091',
stageNumber: 0,
error: {
errorCode: 'CannotParseExternalData',
errorMessage:
'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)',
},
exceptionStackTrace:
'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n',
}, },
], ],
startTime: '2022-10-31T16:16:22.464Z', startTime: '2023-03-27T22:11:24.945Z',
durationMs: 7229, durationMs: 14106,
pendingTasks: 0, pendingTasks: 0,
runningTasks: 2, runningTasks: 2,
}, },
@ -217,13 +184,15 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
{ {
stageNumber: 0, stageNumber: 0,
definition: { definition: {
id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_0', id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_0',
input: [ input: [
{ {
type: 'external', type: 'external',
inputSource: { inputSource: {
type: 'http', type: 'http',
uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'], uris: [
'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json',
],
}, },
inputFormat: { inputFormat: {
type: 'json', type: 'json',
@ -262,45 +231,54 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
finalize: false, finalize: false,
finalizeAggregations: false, finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 10, maxNumTasks: 2,
queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', maxParseExceptions: 2,
queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
scanSignature: scanSignature:
'[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]',
sqlInsertSegmentGranularity: '{"type":"all"}', sqlInsertSegmentGranularity: '"DAY"',
sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765',
sqlReplaceTimeChunks: 'all', sqlReplaceTimeChunks: 'all',
}, },
granularity: { type: 'all' }, granularity: { type: 'all' },
}, },
}, },
signature: [ signature: [
{ name: '__bucket', type: 'LONG' },
{ name: '__boost', type: 'LONG' }, { name: '__boost', type: 'LONG' },
{ name: 'agent_type', type: 'STRING' }, { name: 'agent_type', type: 'STRING' },
{ name: 'v0', type: 'LONG' }, { name: 'v0', type: 'LONG' },
], ],
shuffleSpec: { shuffleSpec: {
type: 'targetSize', type: 'targetSize',
clusterBy: { columns: [{ columnName: '__boost' }] }, clusterBy: {
columns: [
{ columnName: '__bucket', order: 'ASCENDING' },
{ columnName: '__boost', order: 'ASCENDING' },
],
bucketByCount: 1,
},
targetSize: 3000000, targetSize: 3000000,
}, },
maxWorkerCount: 1, maxWorkerCount: 1,
shuffleCheckHasMultipleValues: true, shuffleCheckHasMultipleValues: true,
maxInputBytesPerWorker: 10737418240,
}, },
phase: 'FAILED', phase: 'FAILED',
workerCount: 1, workerCount: 1,
startTime: '2022-10-31T16:16:24.680Z', startTime: '2023-03-27T22:11:25.310Z',
duration: 5013, duration: 13741,
sort: true, sort: true,
}, },
{ {
stageNumber: 1, stageNumber: 1,
definition: { definition: {
id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_1', id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_1',
input: [{ type: 'stage', stage: 0 }], input: [{ type: 'stage', stage: 0 }],
processor: { processor: {
type: 'segmentGenerator', type: 'segmentGenerator',
dataSchema: { dataSchema: {
dataSource: 'kttm_simple', dataSource: 'kttm-blank-lines',
timestampSpec: { column: '__time', format: 'millis', missingValue: null }, timestampSpec: { column: '__time', format: 'millis', missingValue: null },
dimensionsSpec: { dimensionsSpec: {
dimensions: [ dimensions: [
@ -313,6 +291,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
], ],
dimensionExclusions: ['__time'], dimensionExclusions: ['__time'],
includeAllDimensions: false, includeAllDimensions: false,
useSchemaDiscovery: false,
}, },
metricsSpec: [], metricsSpec: [],
granularitySpec: { granularitySpec: {
@ -335,22 +314,24 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport(
}, },
signature: [], signature: [],
maxWorkerCount: 1, maxWorkerCount: 1,
maxInputBytesPerWorker: 10737418240,
}, },
}, },
], ],
counters: { counters: {
'0': { '0': {
'0': { '0': {
input0: { type: 'channel', rows: [237883], totalFiles: [1] }, input0: { type: 'channel', rows: [10], bytes: [7658], files: [1], totalFiles: [1] },
output: { type: 'channel', rows: [141660], bytes: [7685544], frames: [1] }, output: { type: 'channel', rows: [10], bytes: [712], frames: [1] },
sortProgress: { sortProgress: {
type: 'sortProgress', type: 'sortProgress',
totalMergingLevels: -1, totalMergingLevels: 3,
levelToTotalBatches: {}, levelToTotalBatches: { '0': 1, '1': 1, '2': -1 },
levelToMergedBatches: {}, levelToMergedBatches: {},
totalMergersForUltimateLevel: -1, totalMergersForUltimateLevel: -1,
progressDigest: 0.0,
}, },
warnings: { type: 'warnings', CannotParseExternalData: 17 }, warnings: { type: 'warnings', CannotParseExternalData: 3 },
}, },
}, },
}, },

83
web-console/src/druid-models/execution/execution.spec.ts
View File

@ -43,10 +43,10 @@ describe('Execution', () => {
"useLineageBasedSegmentAllocation": true, "useLineageBasedSegmentAllocation": true,
}, },
"dataSource": "kttm_simple", "dataSource": "kttm_simple",
"groupId": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", "groupId": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", "id": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"resource": Object { "resource": Object {
"availabilityGroup": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", "availabilityGroup": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"requiredCapacity": 1, "requiredCapacity": 1,
}, },
"spec": Object { "spec": Object {
@ -80,10 +80,11 @@ describe('Execution', () => {
"finalize": false, "finalize": false,
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all", "sqlReplaceTimeChunks": "all",
}, },
"dataSource": Object { "dataSource": Object {
@ -139,22 +140,24 @@ describe('Execution', () => {
}, },
}, },
"sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL
SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
\\"agent_type\\"
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}', '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}',
'{\\"type\\":\\"json\\"}', '{\\"type\\":\\"json\\"}'
'[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]'
) )
) ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR)
PARTITIONED BY ALL TIME", PARTITIONED BY ALL TIME",
"sqlQueryContext": Object { "sqlQueryContext": Object {
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 2,
"maxParseExceptions": 0, "maxParseExceptions": 0,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all", "sqlReplaceTimeChunks": "all",
}, },
"sqlTypeNames": Array [ "sqlTypeNames": Array [
@ -163,7 +166,7 @@ describe('Execution', () => {
], ],
"type": "query_controller", "type": "query_controller",
}, },
"task": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", "task": "query-b55f3432-7810-4529-80ed-780a926a6f03",
}, },
"capacityInfo": undefined, "capacityInfo": undefined,
"destination": Object { "destination": Object {
@ -176,10 +179,10 @@ describe('Execution', () => {
}, },
"type": "dataSource", "type": "dataSource",
}, },
"duration": 9012, "duration": 28854,
"engine": "sql-msq-task", "engine": "sql-msq-task",
"error": undefined, "error": undefined,
"id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", "id": "query-b55f3432-7810-4529-80ed-780a926a6f03",
"nativeQuery": Object { "nativeQuery": Object {
"columns": Array [ "columns": Array [
"agent_type", "agent_type",
@ -189,10 +192,11 @@ describe('Execution', () => {
"finalize": false, "finalize": false,
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all", "sqlReplaceTimeChunks": "all",
}, },
"dataSource": Object { "dataSource": Object {
@ -244,24 +248,29 @@ describe('Execution', () => {
"queryContext": Object { "queryContext": Object {
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"maxNumTasks": 2,
"maxParseExceptions": 0, "maxParseExceptions": 0,
}, },
"result": undefined, "result": undefined,
"sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL
SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
\\"agent_type\\"
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}', '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}',
'{\\"type\\":\\"json\\"}', '{\\"type\\":\\"json\\"}'
'[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]'
) )
) ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR)
PARTITIONED BY ALL TIME", PARTITIONED BY ALL TIME",
"stages": Stages { "stages": Stages {
"counters": Object { "counters": Object {
"0": Object { "0": Object {
"0": Object { "0": Object {
"input0": Object { "input0": Object {
"bytes": Array [
360464067,
],
"files": Array [ "files": Array [
1, 1,
], ],
@ -299,12 +308,12 @@ describe('Execution', () => {
}, },
"sortProgress": Object { "sortProgress": Object {
"levelToMergedBatches": Object { "levelToMergedBatches": Object {
"0": 2, "0": 1,
"1": 1, "1": 1,
"2": 1, "2": 1,
}, },
"levelToTotalBatches": Object { "levelToTotalBatches": Object {
"0": 2, "0": 1,
"1": 1, "1": 1,
"2": 1, "2": 1,
}, },
@ -329,13 +338,20 @@ describe('Execution', () => {
], ],
"type": "channel", "type": "channel",
}, },
"segmentGenerationProgress": Object {
"rowsMerged": 465346,
"rowsPersisted": 465346,
"rowsProcessed": 465346,
"rowsPushed": 465346,
"type": "segmentGenerationProgress",
},
}, },
}, },
}, },
"stages": Array [ "stages": Array [
Object { Object {
"definition": Object { "definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_0", "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0",
"input": Array [ "input": Array [
Object { Object {
"inputFormat": Object { "inputFormat": Object {
@ -363,6 +379,7 @@ describe('Execution', () => {
"type": "external", "type": "external",
}, },
], ],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1, "maxWorkerCount": 1,
"processor": Object { "processor": Object {
"query": Object { "query": Object {
@ -375,10 +392,11 @@ describe('Execution', () => {
"finalize": false, "finalize": false,
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all", "sqlReplaceTimeChunks": "all",
}, },
"dataSource": Object { "dataSource": Object {
@ -414,6 +432,7 @@ describe('Execution', () => {
"columns": Array [ "columns": Array [
Object { Object {
"columnName": "__boost", "columnName": "__boost",
"order": "ASCENDING",
}, },
], ],
}, },
@ -435,23 +454,24 @@ describe('Execution', () => {
}, },
], ],
}, },
"duration": 4887, "duration": 24236,
"partitionCount": 1, "partitionCount": 1,
"phase": "FINISHED", "phase": "FINISHED",
"sort": true, "sort": true,
"stageNumber": 0, "stageNumber": 0,
"startTime": "2022-10-31T16:11:15.380Z", "startTime": "2023-03-27T22:17:02.792Z",
"workerCount": 1, "workerCount": 1,
}, },
Object { Object {
"definition": Object { "definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_1", "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1",
"input": Array [ "input": Array [
Object { Object {
"stage": 0, "stage": 0,
"type": "stage", "type": "stage",
}, },
], ],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1, "maxWorkerCount": 1,
"processor": Object { "processor": Object {
"columnMappings": Array [ "columnMappings": Array [
@ -479,6 +499,7 @@ describe('Execution', () => {
}, },
], ],
"includeAllDimensions": false, "includeAllDimensions": false,
"useSchemaDiscovery": false,
}, },
"granularitySpec": Object { "granularitySpec": Object {
"intervals": Array [ "intervals": Array [
@ -510,16 +531,16 @@ describe('Execution', () => {
}, },
"signature": Array [], "signature": Array [],
}, },
"duration": 1908, "duration": 4276,
"partitionCount": 1, "partitionCount": 1,
"phase": "FINISHED", "phase": "FINISHED",
"stageNumber": 1, "stageNumber": 1,
"startTime": "2022-10-31T16:11:20.264Z", "startTime": "2023-03-27T22:17:26.978Z",
"workerCount": 1, "workerCount": 1,
}, },
], ],
}, },
"startTime": 2022-10-31T16:11:13.160Z, "startTime": 2023-03-27T22:17:02.401Z,
"status": "SUCCESS", "status": "SUCCESS",
"usageInfo": Object { "usageInfo": Object {
"pendingTasks": 0, "pendingTasks": 0,

44
web-console/src/druid-models/external-config/external-config.ts
View File

@ -22,10 +22,12 @@ import {
F, F,
filterMap, filterMap,
L, L,
SqlColumnDeclaration,
SqlExpression, SqlExpression,
SqlFunction, SqlFunction,
SqlLiteral, SqlLiteral,
SqlStar, SqlStar,
SqlType,
} from 'druid-query-toolkit'; } from 'druid-query-toolkit';
import * as JSONBig from 'json-bigint-native'; import * as JSONBig from 'json-bigint-native';
@ -46,12 +48,7 @@ function joinLinesMax(lines: string[], max: number) {
export interface ExternalConfig { export interface ExternalConfig {
inputSource: InputSource; inputSource: InputSource;
inputFormat: InputFormat; inputFormat: InputFormat;
signature: SignatureColumn[]; signature: readonly SqlColumnDeclaration[];
}
export interface SignatureColumn {
name: string;
type: string;
} }
export function summarizeInputSource(inputSource: InputSource, multiline: boolean): string { export function summarizeInputSource(inputSource: InputSource, multiline: boolean): string {
@ -123,10 +120,9 @@ export function externalConfigToTableExpression(config: ExternalConfig): SqlExpr
return SqlExpression.parse(`TABLE( return SqlExpression.parse(`TABLE(
EXTERN( EXTERN(
${L(JSONBig.stringify(config.inputSource))}, ${L(JSONBig.stringify(config.inputSource))},
${L(JSONBig.stringify(config.inputFormat))}, ${L(JSONBig.stringify(config.inputFormat))}
${L(JSONBig.stringify(config.signature))}
) )
)`); ) EXTEND (${config.signature.join(', ')})`);
} }
export function externalConfigToInitDimensions( export function externalConfigToInitDimensions(
@ -136,9 +132,10 @@ export function externalConfigToInitDimensions(
): SqlExpression[] { ): SqlExpression[] {
return (timeExpression ? [timeExpression.as('__time')] : []) return (timeExpression ? [timeExpression.as('__time')] : [])
.concat( .concat(
filterMap(config.signature, ({ name }, i) => { filterMap(config.signature, (columnDeclaration, i) => {
if (timeExpression && timeExpression.containsColumnName(name)) return; const columnName = columnDeclaration.getColumnName();
return C(name).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(name) as any); if (timeExpression && timeExpression.containsColumnName(columnName)) return;
return C(columnName).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(columnName) as any);
}), }),
) )
.slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS); .slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS);
@ -150,12 +147,12 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig {
} }
const tableFn = query.fromClause?.expressions?.first(); const tableFn = query.fromClause?.expressions?.first();
if (!(tableFn instanceof SqlFunction) || tableFn.functionName !== 'TABLE') { if (!(tableFn instanceof SqlFunction) || tableFn.getEffectiveFunctionName() !== 'TABLE') {
throw new Error(`External FROM must be a TABLE function`); throw new Error(`External FROM must be a TABLE function`);
} }
const externFn = tableFn.getArg(0); const externFn = tableFn.getArg(0);
if (!(externFn instanceof SqlFunction) || externFn.functionName !== 'EXTERN') { if (!(externFn instanceof SqlFunction) || externFn.getEffectiveFunctionName() !== 'EXTERN') {
throw new Error(`Within the TABLE function there must be an extern function`); throw new Error(`Within the TABLE function there must be an extern function`);
} }
@ -175,12 +172,19 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig {
throw new Error(`The second argument to the extern function must be a string embedding JSON`); throw new Error(`The second argument to the extern function must be a string embedding JSON`);
} }
let signature: any; let signature: readonly SqlColumnDeclaration[];
try { const columnDeclarations = tableFn.getColumnDeclarations();
const arg2 = externFn.getArg(2); if (columnDeclarations) {
signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#'); signature = columnDeclarations;
} catch { } else {
throw new Error(`The third argument to the extern function must be a string embedding JSON`); try {
const arg2 = externFn.getArg(2);
signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#').map(
(c: any) => SqlColumnDeclaration.create(c.name, SqlType.fromNativeType(c.type)),
);
} catch {
throw new Error(`The third argument to the extern function must be a string embedding JSON`);
}
} }
return { return {

5
web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
View File

@ -28,10 +28,9 @@ describe('ingest-query-pattern', () => {
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://example.com/data.json.gz"]}', '{"type":"http","uris":["https://example.com/data.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
)) ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" BIGINT, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" BIGINT, "window" VARCHAR))
SELECT SELECT
TIME_PARSE("timestamp") AS __time, TIME_PARSE("timestamp") AS __time,
agent_category, agent_category,

2
web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
View File

@ -133,7 +133,7 @@ function verifyHasOutputName(expression: SqlExpression): void {
} }
export function fitIngestQueryPattern(query: SqlQuery): IngestQueryPattern { export function fitIngestQueryPattern(query: SqlQuery): IngestQueryPattern {
if (query.explainClause) throw new Error(`Can not use EXPLAIN in the data loader flow`); if (query.explain) throw new Error(`Can not use EXPLAIN in the data loader flow`);
if (query.havingClause) throw new Error(`Can not use HAVING in the data loader flow`); if (query.havingClause) throw new Error(`Can not use HAVING in the data loader flow`);
if (query.orderByClause) throw new Error(`Can not USE ORDER BY in the data loader flow`); if (query.orderByClause) throw new Error(`Can not USE ORDER BY in the data loader flow`);
if (query.limitClause) throw new Error(`Can not use LIMIT in the data loader flow`); if (query.limitClause) throw new Error(`Can not use LIMIT in the data loader flow`);

8
web-console/src/druid-models/workbench-query/workbench-query-part.ts
View File

@ -22,7 +22,7 @@ import Hjson from 'hjson';
import * as JSONBig from 'json-bigint-native'; import * as JSONBig from 'json-bigint-native';
import type { ColumnMetadata } from '../../utils'; import type { ColumnMetadata } from '../../utils';
import { compact, filterMap, generate8HexId, sqlTypeFromDruid } from '../../utils'; import { compact, filterMap, generate8HexId } from '../../utils';
import type { LastExecution } from '../execution/execution'; import type { LastExecution } from '../execution/execution';
import { validateLastExecution } from '../execution/execution'; import { validateLastExecution } from '../execution/execution';
import { fitExternalConfigPattern } from '../external-config/external-config'; import { fitExternalConfigPattern } from '../external-config/external-config';
@ -196,9 +196,9 @@ export class WorkbenchQueryPart {
const { queryName, parsedQuery } = this; const { queryName, parsedQuery } = this;
if (queryName && parsedQuery) { if (queryName && parsedQuery) {
try { try {
return fitExternalConfigPattern(parsedQuery).signature.map(({ name, type }) => ({ return fitExternalConfigPattern(parsedQuery).signature.map(columnDeclaration => ({
COLUMN_NAME: name, COLUMN_NAME: columnDeclaration.getColumnName(),
DATA_TYPE: sqlTypeFromDruid(type), DATA_TYPE: columnDeclaration.columnType.getEffectiveType(),
TABLE_NAME: queryName, TABLE_NAME: queryName,
TABLE_SCHEMA: 'druid', TABLE_SCHEMA: 'druid',
})); }));

159
web-console/src/druid-models/workbench-query/workbench-query.spec.ts
View File

@ -88,11 +88,10 @@ describe('WorkbenchQuery', () => {
SELECT *, VAR SELECT *, VAR
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/data/kttm/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY ALL TIME PARTITIONED BY ALL TIME
CLUSTERED BY browser, session CLUSTERED BY browser, session
`; `;
@ -138,10 +137,9 @@ describe('WorkbenchQuery', () => {
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
PARTITIONED BY HOUR PARTITIONED BY HOUR
CLUSTERED BY browser, session CLUSTERED BY browser, session
`); `);
@ -153,10 +151,9 @@ describe('WorkbenchQuery', () => {
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
`); `);
}); });
}); });
@ -447,12 +444,11 @@ describe('WorkbenchQuery', () => {
TIME_PARSE(pickup_datetime) AS __time, TIME_PARSE(pickup_datetime) AS __time,
* *
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type": "local", ...}', '{"type": "local", ...}',
'{"type":"csv", ...}', '{"type":"csv", ...}'
'[{ "name": "cab_type", "type": "string" }, ...]'
)
) )
) EXTEND (cab_type, VARCHAR)
CLUSTERED BY trip_id CLUSTERED BY trip_id
`; `;
@ -483,12 +479,11 @@ describe('WorkbenchQuery', () => {
TIME_PARSE(pickup_datetime) AS __time, TIME_PARSE(pickup_datetime) AS __time,
* *
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type": "local", ...}', '{"type": "local", ...}',
'{"type":"csv", ...}', '{"type":"csv", ...}'
'[{ "name": "cab_type", "type": "string" }, ...]'
)
) )
) EXTEND (cab_type, VARCHAR)
CLUSTERED BY trip_id CLUSTERED BY trip_id
`; `;
@ -567,52 +562,82 @@ describe('WorkbenchQuery', () => {
}); });
describe('#materializeHelpers', () => { describe('#materializeHelpers', () => {
expect( it('works', () => {
WorkbenchQuery.blank() expect(
.changeQueryParts([ WorkbenchQuery.blank()
new WorkbenchQueryPart({ .changeQueryParts([
id: 'aaa', new WorkbenchQueryPart({
queryName: 'kttm_data', id: 'aaa',
queryString: sane` queryName: 'kttm_data',
SELECT * FROM TABLE( queryString: sane`
EXTERN( SELECT * FROM TABLE(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', EXTERN(
'{"type":"json"}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' '{"type":"json"}'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
`, `,
}), }),
new WorkbenchQueryPart({ new WorkbenchQueryPart({
id: 'bbb', id: 'bbb',
queryName: 'country_lookup', queryName: 'country_lookup',
queryString: sane` queryString: sane`
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}', '{"type":"tsv","findColumnsFromHeader":true}'
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]' )
) ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
) `,
`, }),
}), new WorkbenchQueryPart({
new WorkbenchQueryPart({ id: 'ccc',
id: 'ccc', queryName: 'x',
queryName: 'x', queryString: sane`
queryString: sane` SELECT
SELECT os,
os, CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", COUNT(DISTINCT session) AS "unique_sessions"
COUNT(DISTINCT session) AS "unique_sessions" FROM kttm_data
FROM kttm_data LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country
LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country GROUP BY 1, 2
GROUP BY 1, 2 ORDER BY 3 DESC
ORDER BY 3 DESC LIMIT 10
LIMIT 10 `,
`, }),
}), ])
]) .materializeHelpers()
.materializeHelpers(), .getQueryString(),
); ).toEqual(sane`
WITH
"kttm_data" AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)
),
"country_lookup" AS (
SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}'
)
) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
)
(
SELECT
os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",
COUNT(DISTINCT session) AS "unique_sessions"
FROM kttm_data
LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country
GROUP BY 1, 2
ORDER BY 3 DESC
LIMIT 10
)
`);
});
}); });
}); });

190
web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap Normal file
View File

@ -0,0 +1,190 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`spec conversion converts index_hadoop spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "newSource" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"s3","uris":["s3://path"]}',
'{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}'
)
) EXTEND ("event_ts" VARCHAR, "col1" VARCHAR, "col2" VARCHAR, "col3" VARCHAR, "col4" VARCHAR, "field1" DOUBLE, "field2" DOUBLE, "field3" DOUBLE, "field4" VARCHAR, "field5" VARCHAR, "field6" BIGINT, "field7" DOUBLE))
SELECT
TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time,
"col1",
"col2",
"col3",
"col4",
SUM("field1") AS "metric1",
MAX("field2") AS "metric2",
MIN("field3") AS "metric3",
APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4",
APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5",
SUM("field6") AS "metric6",
SUM("field7") AS "metric7"
FROM "source"
WHERE "col2" = 'xxx'
GROUP BY 1, 2, 3, 4, 5
PARTITIONED BY HOUR
`;
exports[`spec conversion converts index_parallel spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia_rollup" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "comment" VARCHAR, "isNew" VARCHAR, "isMinor" VARCHAR, "isAnonymous" VARCHAR, "user" VARCHAR, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "added" BIGINT, "commentLength" BIGINT, "delta" BIGINT, "deltaBucket" BIGINT, "deleted" BIGINT, "page" VARCHAR))
SELECT
TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"comment",
"isNew",
"isMinor",
"isAnonymous",
"user",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
COUNT(*) AS "count",
SUM("added") AS "sum_added",
SUM("commentLength") AS "sum_commentLength",
SUM("delta") AS "sum_delta",
SUM("deltaBucket") AS "sum_deltaBucket",
SUM("deleted") AS "sum_deleted",
APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta"
FROM "source"
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
PARTITIONED BY HOUR
`;
exports[`spec conversion converts index_parallel spec (without rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "event" TYPE('COMPLEX<json>')))
SELECT
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
"event"
FROM "source"
WHERE NOT ("channel" = 'xxx')
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
exports[`spec conversion converts with issue when there is a __time transform 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
exports[`spec conversion converts with issue when there is a dimension transform and strange filter 1`] = `
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}'
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;

201
web-console/src/helpers/spec-conversion.spec.ts
View File

@ -16,10 +16,13 @@
* limitations under the License. * limitations under the License.
*/ */
import { sane } from 'druid-query-toolkit';
import { convertSpecToSql } from './spec-conversion'; import { convertSpecToSql } from './spec-conversion';
expect.addSnapshotSerializer({
test: val => typeof val === 'string',
print: String,
});
describe('spec conversion', () => { describe('spec conversion', () => {
it('converts index_parallel spec (without rollup)', () => { it('converts index_parallel spec (without rollup)', () => {
const converted = convertSpecToSql({ const converted = convertSpecToSql({
@ -96,6 +99,7 @@ describe('spec conversion', () => {
'metroCode', 'metroCode',
'countryIsoCode', 'countryIsoCode',
'regionName', 'regionName',
{ name: 'event', type: 'json' },
], ],
}, },
}, },
@ -116,46 +120,7 @@ describe('spec conversion', () => {
}, },
}); });
expect(converted.queryString).toEqual(sane` expect(converted.queryString).toMatchSnapshot();
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE NOT ("channel" = 'xxx')
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
expect(converted.queryContext).toEqual({ expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
@ -259,45 +224,7 @@ describe('spec conversion', () => {
}, },
}); });
expect(converted.queryString).toEqual(sane` expect(converted.queryString).toMatchSnapshot();
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia_rollup" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"comment","type":"string"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"},{"name":"added","type":"long"},{"name":"commentLength","type":"long"},{"name":"delta","type":"long"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"page","type":"string"}]'
)
))
SELECT
TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"comment",
"isNew",
"isMinor",
"isAnonymous",
"user",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName",
COUNT(*) AS "count",
SUM("added") AS "sum_added",
SUM("commentLength") AS "sum_commentLength",
SUM("delta") AS "sum_delta",
SUM("deltaBucket") AS "sum_deltaBucket",
SUM("deleted") AS "sum_deleted",
APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta"
FROM "source"
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
PARTITIONED BY HOUR
`);
expect(converted.queryContext).toEqual({ expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
@ -421,34 +348,7 @@ describe('spec conversion', () => {
}, },
}); });
expect(converted.queryString).toEqual(sane` expect(converted.queryString).toMatchSnapshot();
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "newSource" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"s3","uris":["s3://path"]}',
'{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}',
'[{"name":"event_ts","type":"string"},{"name":"col1","type":"string"},{"name":"col2","type":"string"},{"name":"col3","type":"string"},{"name":"col4","type":"string"},{"name":"field1","type":"double"},{"name":"field2","type":"double"},{"name":"field3","type":"double"},{"name":"field4","type":"string"},{"name":"field5","type":"string"},{"name":"field6","type":"long"},{"name":"field7","type":"double"}]'
)
))
SELECT
TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time,
"col1",
"col2",
"col3",
"col4",
SUM("field1") AS "metric1",
MAX("field2") AS "metric2",
MIN("field3") AS "metric3",
APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4",
APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5",
SUM("field6") AS "metric6",
SUM("field7") AS "metric7"
FROM "source"
WHERE "col2" = 'xxx'
GROUP BY 1, 2, 3, 4, 5
PARTITIONED BY HOUR
`);
expect(converted.queryContext).toEqual({ expect(converted.queryContext).toEqual({
groupByEnableMultiValueUnnesting: false, groupByEnableMultiValueUnnesting: false,
@ -541,46 +441,7 @@ describe('spec conversion', () => {
}, },
}); });
expect(converted.queryString).toEqual(sane` expect(converted.queryString).toMatchSnapshot();
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
"comment",
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
}); });
it('converts with issue when there is a dimension transform and strange filter', () => { it('converts with issue when there is a dimension transform and strange filter', () => {
@ -671,46 +532,6 @@ describe('spec conversion', () => {
}, },
}); });
expect(converted.queryString).toEqual(sane` expect(converted.queryString).toMatchSnapshot();
-- This SQL query was auto generated from an ingestion spec
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
'{"type":"json"}',
'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
)
))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically converted.
CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
"isRobot",
"channel",
"flags",
"isUnpatrolled",
"page",
"diffUrl",
"added",
REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
"commentLength",
"isNew",
"isMinor",
"delta",
"isAnonymous",
"user",
"deltaBucket",
"deleted",
"namespace",
"cityName",
"countryName",
"regionIsoCode",
"metroCode",
"countryIsoCode",
"regionName"
FROM "source"
WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`);
}); });
}); });

65
web-console/src/helpers/spec-conversion.ts
View File

@ -16,7 +16,15 @@
* limitations under the License. * limitations under the License.
*/ */
import { C, L, RefName, SqlExpression, T } from 'druid-query-toolkit'; import {
C,
L,
RefName,
SqlColumnDeclaration,
SqlExpression,
SqlType,
T,
} from 'druid-query-toolkit';
import * as JSONBig from 'json-bigint-native'; import * as JSONBig from 'json-bigint-native';
import type { import type {
@ -87,20 +95,22 @@ export function convertSpecToSql(spec: any): QueryWithContext {
} }
dimensions = dimensions.map(inflateDimensionSpec); dimensions = dimensions.map(inflateDimensionSpec);
let columns = dimensions.map((d: DimensionSpec) => ({ let columnDeclarations: SqlColumnDeclaration[] = dimensions.map((d: DimensionSpec) =>
name: d.name, SqlColumnDeclaration.create(
type: d.type, d.name,
})); SqlType.fromNativeType(dimensionSpecTypeToNativeDataType(d.type)),
),
);
const metricsSpec = deepGet(spec, 'spec.dataSchema.metricsSpec'); const metricsSpec = deepGet(spec, 'spec.dataSchema.metricsSpec');
if (Array.isArray(metricsSpec)) { if (Array.isArray(metricsSpec)) {
columns = columns.concat( columnDeclarations = columnDeclarations.concat(
filterMap(metricsSpec, metricSpec => filterMap(metricsSpec, metricSpec =>
metricSpec.fieldName metricSpec.fieldName
? { ? SqlColumnDeclaration.create(
name: metricSpec.fieldName, metricSpec.fieldName,
type: metricSpecTypeToDataType(metricSpec.type), SqlType.fromNativeType(metricSpecTypeToNativeDataInputType(metricSpec.type)),
} )
: undefined, : undefined,
), ),
); );
@ -119,42 +129,46 @@ export function convertSpecToSql(spec: any): QueryWithContext {
if (timeTransform) { if (timeTransform) {
timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`; timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
} else { } else {
let timestampColumnType: SqlType;
switch (format) { switch (format) {
case 'auto': case 'auto':
columns.unshift({ name: timestampColumnName, type: 'string' }); timestampColumnType = SqlType.VARCHAR;
timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE TIME_PARSE(${timestampColumn}) END`; timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE TIME_PARSE(${timestampColumn}) END`;
break; break;
case 'iso': case 'iso':
columns.unshift({ name: timestampColumnName, type: 'string' }); timestampColumnType = SqlType.VARCHAR;
timeExpression = `TIME_PARSE(${timestampColumn})`; timeExpression = `TIME_PARSE(${timestampColumn})`;
break; break;
case 'posix': case 'posix':
columns.unshift({ name: timestampColumnName, type: 'long' }); timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`;
break; break;
case 'millis': case 'millis':
columns.unshift({ name: timestampColumnName, type: 'long' }); timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`;
break; break;
case 'micro': case 'micro':
columns.unshift({ name: timestampColumnName, type: 'long' }); timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`;
break; break;
case 'nano': case 'nano':
columns.unshift({ name: timestampColumnName, type: 'long' }); timestampColumnType = SqlType.BIGINT;
timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`;
break; break;
default: default:
columns.unshift({ name: timestampColumnName, type: 'string' }); timestampColumnType = SqlType.VARCHAR;
timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`; timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`;
break; break;
} }
columnDeclarations.unshift(
SqlColumnDeclaration.create(timestampColumnName, timestampColumnType),
);
} }
if (timestampSpec.missingValue) { if (timestampSpec.missingValue) {
@ -238,10 +252,9 @@ export function convertSpecToSql(spec: any): QueryWithContext {
const inputFormat = deepGet(spec, 'spec.ioConfig.inputFormat'); const inputFormat = deepGet(spec, 'spec.ioConfig.inputFormat');
if (!inputFormat) throw new Error(`spec.ioConfig.inputFormat is not defined`); if (!inputFormat) throw new Error(`spec.ioConfig.inputFormat is not defined`);
lines.push( lines.push(
` ${L(JSONBig.stringify(inputFormat))},`, ` ${L(JSONBig.stringify(inputFormat))}`,
` ${L(JSONBig.stringify(columns))}`,
` )`, ` )`,
`))`, `) EXTEND (${columnDeclarations.join(', ')}))`,
); );
} }
@ -362,7 +375,17 @@ const QUERY_GRANULARITY_MAP: Record<string, string> = {
year: `TIME_FLOOR(?, 'P1Y')`, year: `TIME_FLOOR(?, 'P1Y')`,
}; };
function metricSpecTypeToDataType(metricSpecType: string): string { function dimensionSpecTypeToNativeDataType(dimensionSpecType: string): string {
switch (dimensionSpecType) {
case 'json':
return 'COMPLEX<json>';
default:
return dimensionSpecType;
}
}
function metricSpecTypeToNativeDataInputType(metricSpecType: string): string {
const m = /^(long|float|double|string)/.exec(String(metricSpecType)); const m = /^(long|float|double|string)/.exec(String(metricSpecType));
if (m) return m[1]; if (m) return m[1];

18
web-console/src/utils/types.ts
View File

@ -25,24 +25,6 @@ function getEffectiveColumnType(column: Column): string | undefined {
return column.nativeType || column.sqlType; return column.nativeType || column.sqlType;
} }
export function sqlTypeFromDruid(druidType: string): string {
druidType = druidType.toLowerCase();
switch (druidType) {
case 'string':
return 'VARCHAR';
case 'long':
return 'BIGINT';
case 'float':
case 'double':
return druidType.toUpperCase();
default:
return 'COMPLEX';
}
}
export function columnToIcon(column: Column): IconName | undefined { export function columnToIcon(column: Column): IconName | undefined {
const effectiveType = getEffectiveColumnType(column); const effectiveType = getEffectiveColumnType(column);
return effectiveType ? dataTypeToIcon(effectiveType) : undefined; return effectiveType ? dataTypeToIcon(effectiveType) : undefined;

32
web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
View File

@ -42,23 +42,25 @@ function breakdownExpression(expression: SqlExpression): Breakdown {
expression = expression.getUnderlyingExpression(); expression = expression.getUnderlyingExpression();
let nativeType: string | undefined; let nativeType: string | undefined;
if (expression instanceof SqlFunction && expression.getEffectiveFunctionName() === 'CAST') { if (expression instanceof SqlFunction) {
const asType = String(expression.getArgAsString(1)).toUpperCase(); const asType = expression.getCastType();
switch (asType) { if (asType) {
case 'VARCHAR': switch (asType.value.toUpperCase()) {
nativeType = 'string'; case 'VARCHAR':
expression = expression.getArg(0)!; nativeType = 'string';
break; expression = expression.getArg(0)!;
break;
case 'BIGINT': case 'BIGINT':
nativeType = 'long'; nativeType = 'long';
expression = expression.getArg(0)!; expression = expression.getArg(0)!;
break; break;
case 'DOUBLE': case 'DOUBLE':
nativeType = 'double'; nativeType = 'double';
expression = expression.getArg(0)!; expression = expression.getArg(0)!;
break; break;
}
} }
} }

49
web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
View File

@ -451,11 +451,13 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
}, [previewResultState]); }, [previewResultState]);
const unusedColumns = ingestQueryPattern const unusedColumns = ingestQueryPattern
? ingestQueryPattern.mainExternalConfig.signature.filter( ? ingestQueryPattern.mainExternalConfig.signature.filter(columnDeclaration => {
({ name }) => const columnName = columnDeclaration.getColumnName();
!ingestQueryPattern.dimensions.some(d => d.containsColumnName(name)) && return (
!ingestQueryPattern.metrics?.some(m => m.containsColumnName(name)), !ingestQueryPattern.dimensions.some(d => d.containsColumnName(columnName)) &&
) !ingestQueryPattern.metrics?.some(m => m.containsColumnName(columnName))
);
})
: []; : [];
const timeColumn = const timeColumn =
@ -699,23 +701,26 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
)} )}
<MenuDivider /> <MenuDivider />
{unusedColumns.length ? ( {unusedColumns.length ? (
unusedColumns.map((column, i) => ( unusedColumns.map((columnDeclaration, i) => {
<MenuItem const columnName = columnDeclaration.getColumnName();
key={i} return (
icon={dataTypeToIcon(column.type)} <MenuItem
text={column.name} key={i}
onClick={() => { icon={dataTypeToIcon(columnDeclaration.columnType.getNativeType())}
handleQueryAction(q => text={columnName}
q.addSelect( onClick={() => {
C(column.name), handleQueryAction(q =>
ingestQueryPattern.metrics q.addSelect(
? { insertIndex: 'last-grouping', addToGroupBy: 'end' } C(columnName),
: {}, ingestQueryPattern.metrics
), ? { insertIndex: 'last-grouping', addToGroupBy: 'end' }
); : {},
}} ),
/> );
)) }}
/>
);
})
) : ( ) : (
<MenuItem icon={IconNames.BLANK} text="No column suggestions" disabled /> <MenuItem icon={IconNames.BLANK} text="No column suggestions" disabled />
)} )}

35
web-console/src/views/workbench-view/demo-queries.ts
View File

@ -54,10 +54,9 @@ SELECT *
FROM TABLE( FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
) ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)
PARTITIONED BY ALL TIME PARTITIONED BY ALL TIME
`, `,
), ),
@ -85,10 +84,9 @@ WITH kttm_data AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
)) ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR))
SELECT SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -132,18 +130,16 @@ kttm_data AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
)), ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)),
country_lookup AS ( country_lookup AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}', '{"type":"tsv","findColumnsFromHeader":true}'
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
) )
)) ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -190,10 +186,9 @@ country_lookup AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}', '{"type":"tsv","findColumnsFromHeader":true}'
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
) )
)) ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT SELECT
FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time,
@ -261,18 +256,16 @@ kttm_data AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}',
'{"type":"json"}', '{"type":"json"}'
'[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]'
) )
)), ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)),
country_lookup AS ( country_lookup AS (
SELECT * FROM TABLE( SELECT * FROM TABLE(
EXTERN( EXTERN(
'{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}',
'{"type":"tsv","findColumnsFromHeader":true}', '{"type":"tsv","findColumnsFromHeader":true}'
'[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]'
) )
)) ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR))
SELECT SELECT
os, os,
CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country",

1004
web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap
View File

File diff suppressed because it is too large Load Diff

4
web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
View File

@ -16,12 +16,12 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `
</Memo(ExternalLink)> </Memo(ExternalLink)>
: :
</React.Fragment> </React.Fragment>
Too many warnings of type CannotParseExternalData generated (max = 10) Too many warnings of type CannotParseExternalData generated (max = 2)
</p> </p>
<div> <div>
Failed task ID: Failed task ID:
<Memo(ClickToCopy) <Memo(ClickToCopy)
text="query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a" text="query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0"
/> />
<React.Fragment> <React.Fragment>
(on host: (on host:

21
web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap
View File

@ -158,7 +158,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
Array [ Array [
Object { Object {
"definition": Object { "definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_0", "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0",
"input": Array [ "input": Array [
Object { Object {
"inputFormat": Object { "inputFormat": Object {
@ -186,6 +186,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"type": "external", "type": "external",
}, },
], ],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1, "maxWorkerCount": 1,
"processor": Object { "processor": Object {
"query": Object { "query": Object {
@ -198,10 +199,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"finalize": false, "finalize": false,
"finalizeAggregations": false, "finalizeAggregations": false,
"groupByEnableMultiValueUnnesting": false, "groupByEnableMultiValueUnnesting": false,
"queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "maxNumTasks": 2,
"queryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]",
"sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",
"sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03",
"sqlReplaceTimeChunks": "all", "sqlReplaceTimeChunks": "all",
}, },
"dataSource": Object { "dataSource": Object {
@ -237,6 +239,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
"columns": Array [ "columns": Array [
Object { Object {
"columnName": "__boost", "columnName": "__boost",
"order": "ASCENDING",
}, },
], ],
}, },
@ -258,23 +261,24 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
}, },
], ],
}, },
"duration": 4887, "duration": 24236,
"partitionCount": 1, "partitionCount": 1,
"phase": "FINISHED", "phase": "FINISHED",
"sort": true, "sort": true,
"stageNumber": 0, "stageNumber": 0,
"startTime": "2022-10-31T16:11:15.380Z", "startTime": "2023-03-27T22:17:02.792Z",
"workerCount": 1, "workerCount": 1,
}, },
Object { Object {
"definition": Object { "definition": Object {
"id": "b94caff0-f693-47ed-a242-ccf837550383_1", "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1",
"input": Array [ "input": Array [
Object { Object {
"stage": 0, "stage": 0,
"type": "stage", "type": "stage",
}, },
], ],
"maxInputBytesPerWorker": 10737418240,
"maxWorkerCount": 1, "maxWorkerCount": 1,
"processor": Object { "processor": Object {
"columnMappings": Array [ "columnMappings": Array [
@ -302,6 +306,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
}, },
], ],
"includeAllDimensions": false, "includeAllDimensions": false,
"useSchemaDiscovery": false,
}, },
"granularitySpec": Object { "granularitySpec": Object {
"intervals": Array [ "intervals": Array [
@ -333,11 +338,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
}, },
"signature": Array [], "signature": Array [],
}, },
"duration": 1908, "duration": 4276,
"partitionCount": 1, "partitionCount": 1,
"phase": "FINISHED", "phase": "FINISHED",
"stageNumber": 1, "stageNumber": 1,
"startTime": "2022-10-31T16:11:20.264Z", "startTime": "2023-03-27T22:17:26.978Z",
"workerCount": 1, "workerCount": 1,
}, },
] ]

2
web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap
View File

@ -8,7 +8,7 @@ exports[`IngestSuccessPane matches snapshot 1`] = `
465,346 rows inserted into "kttm_simple". 465,346 rows inserted into "kttm_simple".
</p> </p>
<p> <p>
Insert query took 0:00:09. Insert query took 0:00:28.
<span <span
className="action" className="action"
onClick={[Function]} onClick={[Function]}

24
web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
View File

@ -19,11 +19,11 @@
import { Button, Callout, FormGroup, Icon, Intent, Tag } from '@blueprintjs/core'; import { Button, Callout, FormGroup, Icon, Intent, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons'; import { IconNames } from '@blueprintjs/icons';
import type { SqlExpression } from 'druid-query-toolkit'; import type { SqlExpression } from 'druid-query-toolkit';
import { C } from 'druid-query-toolkit'; import { C, SqlColumnDeclaration, SqlType } from 'druid-query-toolkit';
import React, { useState } from 'react'; import React, { useState } from 'react';
import { AutoForm, CenterMessage, LearnMore, Loader } from '../../../components'; import { AutoForm, CenterMessage, LearnMore, Loader } from '../../../components';
import type { InputFormat, InputSource, SignatureColumn } from '../../../druid-models'; import type { InputFormat, InputSource } from '../../../druid-models';
import { import {
guessColumnTypeFromHeaderAndRows, guessColumnTypeFromHeaderAndRows,
guessIsArrayFromHeaderAndRows, guessIsArrayFromHeaderAndRows,
@ -51,7 +51,7 @@ const noop = () => {};
export interface InputFormatAndMore { export interface InputFormatAndMore {
inputFormat: InputFormat; inputFormat: InputFormat;
signature: SignatureColumn[]; signature: SqlColumnDeclaration[];
isArrays: boolean[]; isArrays: boolean[];
timeExpression: SqlExpression | undefined; timeExpression: SqlExpression | undefined;
} }
@ -121,7 +121,7 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
let possibleTimeExpression: PossibleTimeExpression | undefined; let possibleTimeExpression: PossibleTimeExpression | undefined;
if (previewData) { if (previewData) {
possibleTimeExpression = filterMap(previewData.header, column => { possibleTimeExpression = filterMap(previewData.header, column => {
const values = previewData.rows.map(row => row.input[column]); const values = filterMap(previewData.rows, row => row.input?.[column]);
const possibleDruidFormat = possibleDruidFormatForValues(values); const possibleDruidFormat = possibleDruidFormatForValues(values);
if (!possibleDruidFormat) return; if (!possibleDruidFormat) return;
@ -139,14 +139,18 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
previewData && AutoForm.isValidModel(inputFormat, INPUT_FORMAT_FIELDS) previewData && AutoForm.isValidModel(inputFormat, INPUT_FORMAT_FIELDS)
? { ? {
inputFormat, inputFormat,
signature: previewData.header.map(name => ({ signature: previewData.header.map(name =>
name, SqlColumnDeclaration.create(
type: guessColumnTypeFromHeaderAndRows(
previewData,
name, name,
inputFormatOutputsNumericStrings(inputFormat), SqlType.fromNativeType(
guessColumnTypeFromHeaderAndRows(
previewData,
name,
inputFormatOutputsNumericStrings(inputFormat),
),
),
), ),
})), ),
isArrays: previewData.header.map(name => isArrays: previewData.header.map(name =>
guessIsArrayFromHeaderAndRows(previewData, name), guessIsArrayFromHeaderAndRows(previewData, name),
), ),

3
web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
View File

@ -29,6 +29,7 @@ import {
import { IconNames } from '@blueprintjs/icons'; import { IconNames } from '@blueprintjs/icons';
import classNames from 'classnames'; import classNames from 'classnames';
import type { QueryResult } from 'druid-query-toolkit'; import type { QueryResult } from 'druid-query-toolkit';
import { SqlColumnDeclaration } from 'druid-query-toolkit';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useState } from 'react';
import { AutoForm, ExternalLink } from '../../../components'; import { AutoForm, ExternalLink } from '../../../components';
@ -138,7 +139,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS
listDelimiter: BOGUS_LIST_DELIMITER, listDelimiter: BOGUS_LIST_DELIMITER,
columns: ['raw'], columns: ['raw'],
}, },
signature: [{ name: 'raw', type: 'string' }], signature: [SqlColumnDeclaration.create('raw', 'VARCHAR')],
}); });
const result = extractResult( const result = extractResult(

5
web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
View File

@ -165,10 +165,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result
// Casts // Casts
if (selectExpression) { if (selectExpression) {
const underlyingExpression = selectExpression.getUnderlyingExpression(); const underlyingExpression = selectExpression.getUnderlyingExpression();
if ( if (underlyingExpression instanceof SqlFunction && underlyingExpression.getCastType()) {
underlyingExpression instanceof SqlFunction &&
underlyingExpression.getEffectiveFunctionName() === 'CAST'
) {
menuItems.push( menuItems.push(
<MenuItem <MenuItem
key="uncast" key="uncast"

20
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -54,7 +54,7 @@ import {
getUseCache, getUseCache,
summarizeIndexSpec, summarizeIndexSpec,
} from '../../../druid-models'; } from '../../../druid-models';
import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils'; import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
import { MaxTasksButton } from '../max-tasks-button/max-tasks-button'; import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
import './run-panel.scss'; import './run-panel.scss';
@ -107,6 +107,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const maxParseExceptions = getMaxParseExceptions(queryContext); const maxParseExceptions = getMaxParseExceptions(queryContext);
const finalizeAggregations = getFinalizeAggregations(queryContext); const finalizeAggregations = getFinalizeAggregations(queryContext);
const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext); const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast';
const durableShuffleStorage = getDurableShuffleStorage(queryContext); const durableShuffleStorage = getDurableShuffleStorage(queryContext);
const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec'); const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext); const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext);
@ -302,6 +303,23 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v)) changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v))
} }
/> />
<MenuItem
icon={IconNames.INNER_JOIN}
text="Join algorithm"
label={sqlJoinAlgorithm}
>
{['broadcast', 'sortMerge'].map(o => (
<MenuItem
key={o}
icon={tickIcon(sqlJoinAlgorithm === o)}
text={o}
shouldDismissPopover={false}
onClick={() =>
changeQueryContext(deepSet(queryContext, 'sqlJoinAlgorithm', o))
}
/>
))}
</MenuItem>
<MenuItem <MenuItem
icon={IconNames.TH_DERIVED} icon={IconNames.TH_DERIVED}
text="Edit index spec" text="Edit index spec"