diff --git a/licenses.yaml b/licenses.yaml index ee7bb6b1c97..238af9db034 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -5080,7 +5080,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Sebastian McKenzie -version: 7.5.5 +version: 7.12.11 license_file_path: licenses/bin/@babel-code-frame.MIT --- @@ -5099,8 +5099,8 @@ name: "@babel/helper-validator-identifier" license_category: binary module: web-console license_name: MIT License -copyright: Sebastian McKenzie and other contributors -version: 7.14.0 +copyright: The Babel Team +version: 7.19.1 license_file_path: licenses/bin/@babel-helper-validator-identifier.MIT --- @@ -5109,8 +5109,8 @@ name: "@babel/highlight" license_category: binary module: web-console license_name: MIT License -copyright: suchipi -version: 7.5.0 +copyright: The Babel Team +version: 7.18.6 license_file_path: licenses/bin/@babel-highlight.MIT --- @@ -5120,7 +5120,7 @@ license_category: binary module: web-console license_name: MIT License copyright: The Babel Team -version: 7.14.8 +version: 7.21.0 license_file_path: licenses/bin/@babel-runtime.MIT --- @@ -5140,7 +5140,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Palantir Technologies -version: 4.0.1 +version: 4.1.19 --- @@ -5149,7 +5149,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Palantir Technologies -version: 4.0.3 +version: 4.17.6 --- @@ -5158,7 +5158,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Palantir Technologies -version: 4.0.3 +version: 4.4.25 --- @@ -5167,7 +5167,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Palantir Technologies -version: 4.1.1 +version: 4.14.3 --- @@ -5176,7 +5176,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Palantir Technologies -version: 1.0.3 +version: 1.13.10 --- @@ -5285,7 +5285,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Juggle -version: 3.3.1 +version: 3.4.0 --- @@ -5294,7 +5294,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Federico Zivolo -version: 2.11.4 +version: 2.11.6 license_file_path: licenses/bin/@popperjs-core.MIT --- @@ -5319,6 +5319,36 @@ license_file_path: licenses/bin/@types-parse-json.MIT --- +name: "@types/prop-types" +license_category: binary +module: web-console +license_name: MIT License +copyright: Microsoft Corporation. +version: 15.7.3 +license_file_path: licenses/bin/@types-prop-types.MIT + +--- + +name: "@types/react" +license_category: binary +module: web-console +license_name: MIT License +copyright: Microsoft Corporation. +version: 16.14.35 +license_file_path: licenses/bin/@types-react.MIT + +--- + +name: "@types/scheduler" +license_category: binary +module: web-console +license_name: MIT License +copyright: Microsoft Corporation. +version: 0.16.1 +license_file_path: licenses/bin/@types-scheduler.MIT + +--- + name: "ace-builds" license_category: binary module: web-console @@ -5379,6 +5409,16 @@ license_file_path: licenses/bin/babel-plugin-syntax-jsx.MIT --- +name: "call-bind" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.0.2 +license_file_path: licenses/bin/call-bind.MIT + +--- + name: "callsites" license_category: binary module: web-console @@ -5434,7 +5474,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Jed Watson -version: 2.2.6 +version: 2.3.2 license_file_path: licenses/bin/classnames.MIT --- @@ -5634,7 +5674,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Jordan Harband -version: 1.1.3 +version: 1.2.0 license_file_path: licenses/bin/define-properties.MIT --- @@ -5693,7 +5733,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Imply Data -version: 0.17.5 +version: 0.18.2 --- @@ -5727,16 +5767,6 @@ license_file_path: licenses/bin/escape-string-regexp.MIT --- -name: "esutils" -license_category: binary -module: web-console -license_name: BSD-2-Clause License -copyright: Yusuke Suzuki -version: 2.0.3 -license_file_path: licenses/bin/esutils.BSD2 - ---- - name: "file-saver" license_category: binary module: web-console @@ -5787,6 +5817,26 @@ license_file_path: licenses/bin/function-bind.MIT --- +name: "functions-have-names" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.2.3 +license_file_path: licenses/bin/functions-have-names.MIT + +--- + +name: "get-intrinsic" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.2.0 +license_file_path: licenses/bin/get-intrinsic.MIT + +--- + name: "gud" license_category: binary module: web-console @@ -5817,6 +5867,36 @@ license_file_path: licenses/bin/has-own-prop.MIT --- +name: "has-property-descriptors" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.0.0 +license_file_path: licenses/bin/has-property-descriptors.MIT + +--- + +name: "has-symbols" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.0.3 +license_file_path: licenses/bin/has-symbols.MIT + +--- + +name: "has-tostringtag" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.0.0 +license_file_path: licenses/bin/has-tostringtag.MIT + +--- + name: "has" license_category: binary module: web-console @@ -5907,6 +5987,16 @@ license_file_path: licenses/bin/is-arrayish.MIT --- +name: "is-core-module" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 2.10.0 +license_file_path: licenses/bin/is-core-module.MIT + +--- + name: "is-date-object" license_category: binary module: web-console @@ -5922,7 +6012,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Jordan Harband -version: 1.0.4 +version: 1.1.4 license_file_path: licenses/bin/is-regex.MIT --- @@ -6341,7 +6431,7 @@ license_category: binary module: web-console license_name: BSD-3-Clause License copyright: React Community -version: 4.4.2 +version: 4.4.5 license_file_path: licenses/bin/react-transition-group.BSD3 --- @@ -6361,7 +6451,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Ben Newman -version: 0.13.7 +version: 0.13.11 license_file_path: licenses/bin/regenerator-runtime.MIT --- @@ -6371,7 +6461,7 @@ license_category: binary module: web-console license_name: MIT License copyright: Jordan Harband -version: 1.2.0 +version: 1.4.3 license_file_path: licenses/bin/regexp.prototype.flags.MIT --- @@ -6401,7 +6491,7 @@ license_category: binary module: web-console license_name: MIT License copyright: James Halliday -version: 1.12.0 +version: 1.22.1 license_file_path: licenses/bin/resolve.MIT --- @@ -6466,6 +6556,16 @@ license_file_path: licenses/bin/supports-color.MIT --- +name: "supports-preserve-symlinks-flag" +license_category: binary +module: web-console +license_name: MIT License +copyright: Jordan Harband +version: 1.0.0 +license_file_path: licenses/bin/supports-preserve-symlinks-flag.MIT + +--- + name: "tiny-invariant" license_category: binary module: web-console diff --git a/licenses/bin/@types-prop-types.MIT b/licenses/bin/@types-prop-types.MIT new file mode 100644 index 00000000000..4b1ad51b2f0 --- /dev/null +++ b/licenses/bin/@types-prop-types.MIT @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/licenses/bin/@types-react.MIT b/licenses/bin/@types-react.MIT new file mode 100755 index 00000000000..9e841e7a26e --- /dev/null +++ b/licenses/bin/@types-react.MIT @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/licenses/bin/@types-scheduler.MIT b/licenses/bin/@types-scheduler.MIT new file mode 100644 index 00000000000..4b1ad51b2f0 --- /dev/null +++ b/licenses/bin/@types-scheduler.MIT @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/licenses/bin/call-bind.MIT b/licenses/bin/call-bind.MIT new file mode 100644 index 00000000000..48f05d01d0a --- /dev/null +++ b/licenses/bin/call-bind.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/classnames.MIT b/licenses/bin/classnames.MIT index d8e561e950e..4117bfae030 100644 --- a/licenses/bin/classnames.MIT +++ b/licenses/bin/classnames.MIT @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2017 Jed Watson +Copyright (c) 2018 Jed Watson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/licenses/bin/functions-have-names.MIT b/licenses/bin/functions-have-names.MIT new file mode 100644 index 00000000000..3900dd7e2ff --- /dev/null +++ b/licenses/bin/functions-have-names.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/get-intrinsic.MIT b/licenses/bin/get-intrinsic.MIT new file mode 100644 index 00000000000..48f05d01d0a --- /dev/null +++ b/licenses/bin/get-intrinsic.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/has-property-descriptors.MIT b/licenses/bin/has-property-descriptors.MIT new file mode 100644 index 00000000000..2e7b9a3eacf --- /dev/null +++ b/licenses/bin/has-property-descriptors.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Inspect JS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/has-symbols.MIT b/licenses/bin/has-symbols.MIT new file mode 100644 index 00000000000..df31cbf3c06 --- /dev/null +++ b/licenses/bin/has-symbols.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/has-tostringtag.MIT b/licenses/bin/has-tostringtag.MIT new file mode 100644 index 00000000000..7948bc02a79 --- /dev/null +++ b/licenses/bin/has-tostringtag.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Inspect JS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/bin/is-core-module.MIT b/licenses/bin/is-core-module.MIT new file mode 100644 index 00000000000..2e502872a74 --- /dev/null +++ b/licenses/bin/is-core-module.MIT @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Dave Justice + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses/bin/regexp.prototype.flags.MIT b/licenses/bin/regexp.prototype.flags.MIT index 9a8e1e0e62e..e5956a86be2 100644 --- a/licenses/bin/regexp.prototype.flags.MIT +++ b/licenses/bin/regexp.prototype.flags.MIT @@ -18,4 +18,5 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. + diff --git a/licenses/bin/supports-preserve-symlinks-flag.MIT b/licenses/bin/supports-preserve-symlinks-flag.MIT new file mode 100644 index 00000000000..2e7b9a3eacf --- /dev/null +++ b/licenses/bin/supports-preserve-symlinks-flag.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Inspect JS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/web-console/e2e-tests/multi-stage-query.spec.ts b/web-console/e2e-tests/multi-stage-query.spec.ts index 8c0678ce3f3..7e4168392f8 100644 --- a/web-console/e2e-tests/multi-stage-query.spec.ts +++ b/web-console/e2e-tests/multi-stage-query.spec.ts @@ -52,10 +52,9 @@ FROM TABLE( '{"type":"local","filter":"wikiticker-2015-09-12-sampled.json.gz","baseDir":${JSON.stringify( DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, )}}', - '{"type":"json"}', - '[{"name":"channel","type":"string"}]' + '{"type":"json"}' ) -)) +) EXTEND (channel VARCHAR)) SELECT channel, CAST(COUNT(*) AS VARCHAR) AS "CountString" diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 0f4a5ed8b49..b9aefc10dea 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -22,7 +22,7 @@ "d3-axis": "^2.1.0", "d3-scale": "^3.3.0", "d3-selection": "^2.0.0", - "druid-query-toolkit": "^0.17.5", + "druid-query-toolkit": "^0.18.2", "file-saver": "^2.0.2", "follow-redirects": "^1.14.7", "fontsource-open-sans": "^3.0.9", @@ -8211,9 +8211,9 @@ } }, "node_modules/druid-query-toolkit": { - "version": "0.17.5", - "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz", - "integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==", + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz", + "integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==", "dependencies": { "tslib": "^2.3.1" }, @@ -32625,9 +32625,9 @@ } }, "druid-query-toolkit": { - "version": "0.17.5", - "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.17.5.tgz", - "integrity": "sha512-N+kqu6xy2Gd3zQwNbBxQXG+qjU7jzbCXvE84uxsIh5gxRbiKAEOsyWtKrWF/DZZ91g/WSIMQih5klkmmcjlVfQ==", + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-0.18.2.tgz", + "integrity": "sha512-MUqTm6wW+clI0pVeK9RIdB8svWK6mu44zsAw8BSVZYYKchigbBzTgwJe0vAYFBfR0TPjD1gJl62pSw4g0F14fQ==", "requires": { "tslib": "^2.3.1" } diff --git a/web-console/package.json b/web-console/package.json index f3dbb8b3461..18e99efec43 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -17,9 +17,6 @@ "testMatch": [ "**/?(*.)+(spec).ts?(x)" ], - "snapshotSerializers": [ - "enzyme-to-json/serializer" - ], "setupFilesAfterEnv": [ "src/setup-tests.ts" ] @@ -79,7 +76,7 @@ "d3-axis": "^2.1.0", "d3-scale": "^3.3.0", "d3-selection": "^2.0.0", - "druid-query-toolkit": "^0.17.5", + "druid-query-toolkit": "^0.18.2", "file-saver": "^2.0.2", "follow-redirects": "^1.14.7", "fontsource-open-sans": "^3.0.9", diff --git a/web-console/script/create-sql-docs.js b/web-console/script/create-sql-docs.js index 13ed438915b..91dfceffad4 100755 --- a/web-console/script/create-sql-docs.js +++ b/web-console/script/create-sql-docs.js @@ -23,9 +23,16 @@ const snarkdown = require('snarkdown'); const writefile = 'lib/sql-docs.js'; -const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162; +const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 164; const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14; +const initialFunctionDocs = { + TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an external.')]], + EXTERN: [ + ['inputSource, inputFormat, rowSignature?', convertMarkdownToHtml('Reads external data')], + ], +}; + function hasHtmlTags(str) { return /<(a|br|span|div|p|code)\/?>/.test(str); } @@ -67,7 +74,7 @@ const readDoc = async () => { const lines = data.split('\n'); - const functionDocs = {}; + const functionDocs = initialFunctionDocs; const dataTypeDocs = {}; for (let line of lines) { const functionMatch = line.match(/^\|\s*`(\w+)\(([^|]*)\)`\s*\|([^|]+)\|(?:([^|]+)\|)?$/); diff --git a/web-console/src/bootstrap/ace.scss b/web-console/src/bootstrap/ace.scss index e764348a8e3..9b1b5bf7410 100644 --- a/web-console/src/bootstrap/ace.scss +++ b/web-console/src/bootstrap/ace.scss @@ -35,6 +35,10 @@ text-decoration: underline; text-decoration-style: wavy; } + + .ace_storage.ace_type { + color: #27c923; + } // END: Custom code styles &.no-background { diff --git a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts index 64bb2201d8f..aff948e8ee2 100644 --- a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts @@ -19,26 +19,33 @@ import { Execution } from './execution'; /* -For query: +===== Query ===== REPLACE INTO "kttm_simple" OVERWRITE ALL -SELECT TIME_PARSE("timestamp") AS "__time", agent_type +SELECT + TIME_PARSE("timestamp") AS "__time", + "agent_type" FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]' + '{"type":"json"}' ) -) +) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) PARTITIONED BY ALL TIME + +===== Context ===== + +{ + "maxNumTasks": 2 +} */ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( { - task: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', + task: 'query-b55f3432-7810-4529-80ed-780a926a6f03', payload: { type: 'query_controller', - id: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', + id: 'query-b55f3432-7810-4529-80ed-780a926a6f03', spec: { query: { queryType: 'scan', @@ -78,10 +85,11 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( finalize: false, finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, - queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + maxNumTasks: 2, + queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', sqlReplaceTimeChunks: 'all', }, granularity: { type: 'all' }, @@ -100,22 +108,23 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, }, sqlQuery: - 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL TIME', + 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME', sqlQueryContext: { finalizeAggregations: false, maxParseExceptions: 0, - sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', groupByEnableMultiValueUnnesting: false, sqlInsertSegmentGranularity: '{"type":"all"}', + maxNumTasks: 2, sqlReplaceTimeChunks: 'all', - queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', }, sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, - groupId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', + groupId: 'query-b55f3432-7810-4529-80ed-780a926a6f03', dataSource: 'kttm_simple', resource: { - availabilityGroup: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', + availabilityGroup: 'query-b55f3432-7810-4529-80ed-780a926a6f03', requiredCapacity: 1, }, }, @@ -124,12 +133,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( { multiStageQuery: { type: 'multiStageQuery', - taskId: 'query-09af0c1e-1c0f-4539-917c-b0458849d0d9', + taskId: 'query-b55f3432-7810-4529-80ed-780a926a6f03', payload: { status: { status: 'SUCCESS', - startTime: '2022-10-31T16:11:13.160Z', - durationMs: 9012, + startTime: '2023-03-27T22:17:02.401Z', + durationMs: 28854, pendingTasks: 0, runningTasks: 2, }, @@ -137,7 +146,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( { stageNumber: 0, definition: { - id: 'b94caff0-f693-47ed-a242-ccf837550383_0', + id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0', input: [ { type: 'external', @@ -184,11 +193,12 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( finalize: false, finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, - queryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + maxNumTasks: 2, + queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '09af0c1e-1c0f-4539-917c-b0458849d0d9', + sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', sqlReplaceTimeChunks: 'all', }, granularity: { type: 'all' }, @@ -201,23 +211,24 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( ], shuffleSpec: { type: 'targetSize', - clusterBy: { columns: [{ columnName: '__boost' }] }, + clusterBy: { columns: [{ columnName: '__boost', order: 'ASCENDING' }] }, targetSize: 3000000, }, maxWorkerCount: 1, shuffleCheckHasMultipleValues: true, + maxInputBytesPerWorker: 10737418240, }, phase: 'FINISHED', workerCount: 1, partitionCount: 1, - startTime: '2022-10-31T16:11:15.380Z', - duration: 4887, + startTime: '2023-03-27T22:17:02.792Z', + duration: 24236, sort: true, }, { stageNumber: 1, definition: { - id: 'b94caff0-f693-47ed-a242-ccf837550383_1', + id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1', input: [{ type: 'stage', stage: 0 }], processor: { type: 'segmentGenerator', @@ -235,6 +246,7 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( ], dimensionExclusions: ['__time'], includeAllDimensions: false, + useSchemaDiscovery: false, }, metricsSpec: [], granularitySpec: { @@ -257,32 +269,48 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( }, signature: [], maxWorkerCount: 1, + maxInputBytesPerWorker: 10737418240, }, phase: 'FINISHED', workerCount: 1, partitionCount: 1, - startTime: '2022-10-31T16:11:20.264Z', - duration: 1908, + startTime: '2023-03-27T22:17:26.978Z', + duration: 4276, }, ], counters: { '0': { '0': { - input0: { type: 'channel', rows: [465346], files: [1], totalFiles: [1] }, + input0: { + type: 'channel', + rows: [465346], + bytes: [360464067], + files: [1], + totalFiles: [1], + }, output: { type: 'channel', rows: [465346], bytes: [25430674], frames: [4] }, shuffle: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] }, sortProgress: { type: 'sortProgress', totalMergingLevels: 3, - levelToTotalBatches: { '0': 2, '1': 1, '2': 1 }, - levelToMergedBatches: { '0': 2, '1': 1, '2': 1 }, + levelToTotalBatches: { '0': 1, '1': 1, '2': 1 }, + levelToMergedBatches: { '0': 1, '1': 1, '2': 1 }, totalMergersForUltimateLevel: 1, progressDigest: 1.0, }, }, }, '1': { - '0': { input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] } }, + '0': { + input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] }, + segmentGenerationProgress: { + type: 'segmentGenerationProgress', + rowsProcessed: 465346, + rowsPersisted: 465346, + rowsMerged: 465346, + rowsPushed: 465346, + }, + }, }, }, }, diff --git a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts index 42fbde8dbf3..485256c6b4a 100644 --- a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts @@ -21,30 +21,32 @@ import { Execution } from './execution'; /* ===== Query ===== -REPLACE INTO "kttm_simple" OVERWRITE ALL -SELECT TIME_PARSE("timestamp") AS "__time", agent_type +REPLACE INTO "kttm-blank-lines" OVERWRITE ALL +SELECT + TIME_PARSE("timestamp") AS "__time", + "agent_type" FROM TABLE( EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]' + '{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}', + '{"type":"json"}' ) -) -PARTITIONED BY ALL +) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) +PARTITIONED BY DAY ===== Context ===== { - "maxParseExceptions": 10 + "maxParseExceptions": 2, + "maxNumTasks": 2 } */ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( { - task: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + task: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', payload: { type: 'query_controller', - id: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + id: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', spec: { query: { queryType: 'scan', @@ -52,7 +54,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( type: 'external', inputSource: { type: 'http', - uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'], + uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'], }, inputFormat: { type: 'json', @@ -84,11 +86,12 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( finalize: false, finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, - maxParseExceptions: 10, - queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + maxNumTasks: 2, + maxParseExceptions: 2, + queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + sqlInsertSegmentGranularity: '"DAY"', + sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', sqlReplaceTimeChunks: 'all', }, granularity: { type: 'all' }, @@ -99,30 +102,31 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( ], destination: { type: 'dataSource', - dataSource: 'kttm_simple', - segmentGranularity: { type: 'all' }, + dataSource: 'kttm-blank-lines', + segmentGranularity: 'DAY', replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], }, assignmentStrategy: 'max', tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, }, sqlQuery: - 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT TIME_PARSE("timestamp") AS "__time", agent_type\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\',\n \'[{"name":"timestamp","type":"string"},{"name":"agent_type","type":"string"}]\'\n )\n)\nPARTITIONED BY ALL', + 'REPLACE INTO "kttm-blank-lines" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY DAY', sqlQueryContext: { - maxParseExceptions: 10, + maxParseExceptions: 2, finalizeAggregations: false, - sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', groupByEnableMultiValueUnnesting: false, - sqlInsertSegmentGranularity: '{"type":"all"}', + sqlInsertSegmentGranularity: '"DAY"', + maxNumTasks: 2, sqlReplaceTimeChunks: 'all', - queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', }, sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, - groupId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', - dataSource: 'kttm_simple', + groupId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', + dataSource: 'kttm-blank-lines', resource: { - availabilityGroup: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + availabilityGroup: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', requiredCapacity: 1, }, }, @@ -131,85 +135,48 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( { multiStageQuery: { type: 'multiStageQuery', - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', payload: { status: { status: 'FAILED', errorReport: { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', host: 'localhost', error: { errorCode: 'TooManyWarnings', - maxWarnings: 10, + maxWarnings: 2, rootErrorCode: 'CannotParseExternalData', - errorMessage: - 'Too many warnings of type CannotParseExternalData generated (max = 10)', + errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)', }, }, warnings: [ { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', - host: 'localhost:8091', + taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', + host: 'localhost:8101', stageNumber: 0, error: { errorCode: 'CannotParseExternalData', errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)', + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)', }, exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 13588, Line: 13588)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', }, { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', - host: 'localhost:8091', + taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', + host: 'localhost:8101', stageNumber: 0, error: { errorCode: 'CannotParseExternalData', errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)', + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)', }, exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 27029, Line: 27030)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', - }, - { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', - host: 'localhost:8091', - stageNumber: 0, - error: { - errorCode: 'CannotParseExternalData', - errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)', - }, - exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 42034, Line: 42036)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', - }, - { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', - host: 'localhost:8091', - stageNumber: 0, - error: { - errorCode: 'CannotParseExternalData', - errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)', - }, - exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 54912, Line: 54915)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', - }, - { - taskId: 'query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a-worker0', - host: 'localhost:8091', - stageNumber: 0, - error: { - errorCode: 'CannotParseExternalData', - errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)', - }, - exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz, Record: 63995, Line: 63999)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:179)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:245)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:172)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:137)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:70)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:666)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:69)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', }, ], - startTime: '2022-10-31T16:16:22.464Z', - durationMs: 7229, + startTime: '2023-03-27T22:11:24.945Z', + durationMs: 14106, pendingTasks: 0, runningTasks: 2, }, @@ -217,13 +184,15 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( { stageNumber: 0, definition: { - id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_0', + id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_0', input: [ { type: 'external', inputSource: { type: 'http', - uris: ['https://static.imply.io/example-data/kttm/kttm-2019-08-25.json.gz'], + uris: [ + 'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json', + ], }, inputFormat: { type: 'json', @@ -262,45 +231,54 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( finalize: false, finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, - maxParseExceptions: 10, - queryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + maxNumTasks: 2, + maxParseExceptions: 2, + queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: '955fa5a2-0ae6-4912-bc39-cc9fc012de5a', + sqlInsertSegmentGranularity: '"DAY"', + sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', sqlReplaceTimeChunks: 'all', }, granularity: { type: 'all' }, }, }, signature: [ + { name: '__bucket', type: 'LONG' }, { name: '__boost', type: 'LONG' }, { name: 'agent_type', type: 'STRING' }, { name: 'v0', type: 'LONG' }, ], shuffleSpec: { type: 'targetSize', - clusterBy: { columns: [{ columnName: '__boost' }] }, + clusterBy: { + columns: [ + { columnName: '__bucket', order: 'ASCENDING' }, + { columnName: '__boost', order: 'ASCENDING' }, + ], + bucketByCount: 1, + }, targetSize: 3000000, }, maxWorkerCount: 1, shuffleCheckHasMultipleValues: true, + maxInputBytesPerWorker: 10737418240, }, phase: 'FAILED', workerCount: 1, - startTime: '2022-10-31T16:16:24.680Z', - duration: 5013, + startTime: '2023-03-27T22:11:25.310Z', + duration: 13741, sort: true, }, { stageNumber: 1, definition: { - id: 'fc6c3715-9211-4f6e-9d7a-a7e6a8e6b297_1', + id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_1', input: [{ type: 'stage', stage: 0 }], processor: { type: 'segmentGenerator', dataSchema: { - dataSource: 'kttm_simple', + dataSource: 'kttm-blank-lines', timestampSpec: { column: '__time', format: 'millis', missingValue: null }, dimensionsSpec: { dimensions: [ @@ -313,6 +291,7 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( ], dimensionExclusions: ['__time'], includeAllDimensions: false, + useSchemaDiscovery: false, }, metricsSpec: [], granularitySpec: { @@ -335,22 +314,24 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( }, signature: [], maxWorkerCount: 1, + maxInputBytesPerWorker: 10737418240, }, }, ], counters: { '0': { '0': { - input0: { type: 'channel', rows: [237883], totalFiles: [1] }, - output: { type: 'channel', rows: [141660], bytes: [7685544], frames: [1] }, + input0: { type: 'channel', rows: [10], bytes: [7658], files: [1], totalFiles: [1] }, + output: { type: 'channel', rows: [10], bytes: [712], frames: [1] }, sortProgress: { type: 'sortProgress', - totalMergingLevels: -1, - levelToTotalBatches: {}, + totalMergingLevels: 3, + levelToTotalBatches: { '0': 1, '1': 1, '2': -1 }, levelToMergedBatches: {}, totalMergersForUltimateLevel: -1, + progressDigest: 0.0, }, - warnings: { type: 'warnings', CannotParseExternalData: 17 }, + warnings: { type: 'warnings', CannotParseExternalData: 3 }, }, }, }, diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index 43f4301c4ce..f2d4b1e3038 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -43,10 +43,10 @@ describe('Execution', () => { "useLineageBasedSegmentAllocation": true, }, "dataSource": "kttm_simple", - "groupId": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", - "id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", + "groupId": "query-b55f3432-7810-4529-80ed-780a926a6f03", + "id": "query-b55f3432-7810-4529-80ed-780a926a6f03", "resource": Object { - "availabilityGroup": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", + "availabilityGroup": "query-b55f3432-7810-4529-80ed-780a926a6f03", "requiredCapacity": 1, }, "spec": Object { @@ -80,10 +80,11 @@ describe('Execution', () => { "finalize": false, "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, - "queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "maxNumTasks": 2, + "queryId": "b55f3432-7810-4529-80ed-780a926a6f03", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlReplaceTimeChunks": "all", }, "dataSource": Object { @@ -139,22 +140,24 @@ describe('Execution', () => { }, }, "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL - SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type + SELECT + TIME_PARSE(\\"timestamp\\") AS \\"__time\\", + \\"agent_type\\" FROM TABLE( EXTERN( '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}', - '{\\"type\\":\\"json\\"}', - '[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]' + '{\\"type\\":\\"json\\"}' ) - ) + ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY ALL TIME", "sqlQueryContext": Object { "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, + "maxNumTasks": 2, "maxParseExceptions": 0, - "queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "queryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlReplaceTimeChunks": "all", }, "sqlTypeNames": Array [ @@ -163,7 +166,7 @@ describe('Execution', () => { ], "type": "query_controller", }, - "task": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", + "task": "query-b55f3432-7810-4529-80ed-780a926a6f03", }, "capacityInfo": undefined, "destination": Object { @@ -176,10 +179,10 @@ describe('Execution', () => { }, "type": "dataSource", }, - "duration": 9012, + "duration": 28854, "engine": "sql-msq-task", "error": undefined, - "id": "query-09af0c1e-1c0f-4539-917c-b0458849d0d9", + "id": "query-b55f3432-7810-4529-80ed-780a926a6f03", "nativeQuery": Object { "columns": Array [ "agent_type", @@ -189,10 +192,11 @@ describe('Execution', () => { "finalize": false, "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, - "queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "maxNumTasks": 2, + "queryId": "b55f3432-7810-4529-80ed-780a926a6f03", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlReplaceTimeChunks": "all", }, "dataSource": Object { @@ -244,24 +248,29 @@ describe('Execution', () => { "queryContext": Object { "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, + "maxNumTasks": 2, "maxParseExceptions": 0, }, "result": undefined, "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL - SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", agent_type + SELECT + TIME_PARSE(\\"timestamp\\") AS \\"__time\\", + \\"agent_type\\" FROM TABLE( EXTERN( '{\\"type\\":\\"http\\",\\"uris\\":[\\"https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz\\"]}', - '{\\"type\\":\\"json\\"}', - '[{\\"name\\":\\"timestamp\\",\\"type\\":\\"string\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"string\\"}]' + '{\\"type\\":\\"json\\"}' ) - ) + ) EXTEND (\\"timestamp\\" VARCHAR, \\"agent_type\\" VARCHAR) PARTITIONED BY ALL TIME", "stages": Stages { "counters": Object { "0": Object { "0": Object { "input0": Object { + "bytes": Array [ + 360464067, + ], "files": Array [ 1, ], @@ -299,12 +308,12 @@ describe('Execution', () => { }, "sortProgress": Object { "levelToMergedBatches": Object { - "0": 2, + "0": 1, "1": 1, "2": 1, }, "levelToTotalBatches": Object { - "0": 2, + "0": 1, "1": 1, "2": 1, }, @@ -329,13 +338,20 @@ describe('Execution', () => { ], "type": "channel", }, + "segmentGenerationProgress": Object { + "rowsMerged": 465346, + "rowsPersisted": 465346, + "rowsProcessed": 465346, + "rowsPushed": 465346, + "type": "segmentGenerationProgress", + }, }, }, }, "stages": Array [ Object { "definition": Object { - "id": "b94caff0-f693-47ed-a242-ccf837550383_0", + "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0", "input": Array [ Object { "inputFormat": Object { @@ -363,6 +379,7 @@ describe('Execution', () => { "type": "external", }, ], + "maxInputBytesPerWorker": 10737418240, "maxWorkerCount": 1, "processor": Object { "query": Object { @@ -375,10 +392,11 @@ describe('Execution', () => { "finalize": false, "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, - "queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "maxNumTasks": 2, + "queryId": "b55f3432-7810-4529-80ed-780a926a6f03", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlReplaceTimeChunks": "all", }, "dataSource": Object { @@ -414,6 +432,7 @@ describe('Execution', () => { "columns": Array [ Object { "columnName": "__boost", + "order": "ASCENDING", }, ], }, @@ -435,23 +454,24 @@ describe('Execution', () => { }, ], }, - "duration": 4887, + "duration": 24236, "partitionCount": 1, "phase": "FINISHED", "sort": true, "stageNumber": 0, - "startTime": "2022-10-31T16:11:15.380Z", + "startTime": "2023-03-27T22:17:02.792Z", "workerCount": 1, }, Object { "definition": Object { - "id": "b94caff0-f693-47ed-a242-ccf837550383_1", + "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1", "input": Array [ Object { "stage": 0, "type": "stage", }, ], + "maxInputBytesPerWorker": 10737418240, "maxWorkerCount": 1, "processor": Object { "columnMappings": Array [ @@ -479,6 +499,7 @@ describe('Execution', () => { }, ], "includeAllDimensions": false, + "useSchemaDiscovery": false, }, "granularitySpec": Object { "intervals": Array [ @@ -510,16 +531,16 @@ describe('Execution', () => { }, "signature": Array [], }, - "duration": 1908, + "duration": 4276, "partitionCount": 1, "phase": "FINISHED", "stageNumber": 1, - "startTime": "2022-10-31T16:11:20.264Z", + "startTime": "2023-03-27T22:17:26.978Z", "workerCount": 1, }, ], }, - "startTime": 2022-10-31T16:11:13.160Z, + "startTime": 2023-03-27T22:17:02.401Z, "status": "SUCCESS", "usageInfo": Object { "pendingTasks": 0, diff --git a/web-console/src/druid-models/external-config/external-config.ts b/web-console/src/druid-models/external-config/external-config.ts index 012c7eef479..b45e413ade5 100644 --- a/web-console/src/druid-models/external-config/external-config.ts +++ b/web-console/src/druid-models/external-config/external-config.ts @@ -22,10 +22,12 @@ import { F, filterMap, L, + SqlColumnDeclaration, SqlExpression, SqlFunction, SqlLiteral, SqlStar, + SqlType, } from 'druid-query-toolkit'; import * as JSONBig from 'json-bigint-native'; @@ -46,12 +48,7 @@ function joinLinesMax(lines: string[], max: number) { export interface ExternalConfig { inputSource: InputSource; inputFormat: InputFormat; - signature: SignatureColumn[]; -} - -export interface SignatureColumn { - name: string; - type: string; + signature: readonly SqlColumnDeclaration[]; } export function summarizeInputSource(inputSource: InputSource, multiline: boolean): string { @@ -123,10 +120,9 @@ export function externalConfigToTableExpression(config: ExternalConfig): SqlExpr return SqlExpression.parse(`TABLE( EXTERN( ${L(JSONBig.stringify(config.inputSource))}, - ${L(JSONBig.stringify(config.inputFormat))}, - ${L(JSONBig.stringify(config.signature))} + ${L(JSONBig.stringify(config.inputFormat))} ) -)`); +) EXTEND (${config.signature.join(', ')})`); } export function externalConfigToInitDimensions( @@ -136,9 +132,10 @@ export function externalConfigToInitDimensions( ): SqlExpression[] { return (timeExpression ? [timeExpression.as('__time')] : []) .concat( - filterMap(config.signature, ({ name }, i) => { - if (timeExpression && timeExpression.containsColumnName(name)) return; - return C(name).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(name) as any); + filterMap(config.signature, (columnDeclaration, i) => { + const columnName = columnDeclaration.getColumnName(); + if (timeExpression && timeExpression.containsColumnName(columnName)) return; + return C(columnName).applyIf(isArrays[i], ex => F('MV_TO_ARRAY', ex).as(columnName) as any); }), ) .slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS); @@ -150,12 +147,12 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig { } const tableFn = query.fromClause?.expressions?.first(); - if (!(tableFn instanceof SqlFunction) || tableFn.functionName !== 'TABLE') { + if (!(tableFn instanceof SqlFunction) || tableFn.getEffectiveFunctionName() !== 'TABLE') { throw new Error(`External FROM must be a TABLE function`); } const externFn = tableFn.getArg(0); - if (!(externFn instanceof SqlFunction) || externFn.functionName !== 'EXTERN') { + if (!(externFn instanceof SqlFunction) || externFn.getEffectiveFunctionName() !== 'EXTERN') { throw new Error(`Within the TABLE function there must be an extern function`); } @@ -175,12 +172,19 @@ export function fitExternalConfigPattern(query: SqlQuery): ExternalConfig { throw new Error(`The second argument to the extern function must be a string embedding JSON`); } - let signature: any; - try { - const arg2 = externFn.getArg(2); - signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#'); - } catch { - throw new Error(`The third argument to the extern function must be a string embedding JSON`); + let signature: readonly SqlColumnDeclaration[]; + const columnDeclarations = tableFn.getColumnDeclarations(); + if (columnDeclarations) { + signature = columnDeclarations; + } else { + try { + const arg2 = externFn.getArg(2); + signature = JSONBig.parse(arg2 instanceof SqlLiteral ? String(arg2.value) : '#').map( + (c: any) => SqlColumnDeclaration.create(c.name, SqlType.fromNativeType(c.type)), + ); + } catch { + throw new Error(`The third argument to the extern function must be a string embedding JSON`); + } } return { diff --git a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts index f7827ea4cd1..003739a5c97 100644 --- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts +++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts @@ -28,10 +28,9 @@ describe('ingest-query-pattern', () => { FROM TABLE( EXTERN( '{"type":"http","uris":["https://example.com/data.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"json"}' ) - )) + ) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" BIGINT, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" BIGINT, "window" VARCHAR)) SELECT TIME_PARSE("timestamp") AS __time, agent_category, diff --git a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts index 9cb7d06725b..029c4dd29a7 100644 --- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts +++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts @@ -133,7 +133,7 @@ function verifyHasOutputName(expression: SqlExpression): void { } export function fitIngestQueryPattern(query: SqlQuery): IngestQueryPattern { - if (query.explainClause) throw new Error(`Can not use EXPLAIN in the data loader flow`); + if (query.explain) throw new Error(`Can not use EXPLAIN in the data loader flow`); if (query.havingClause) throw new Error(`Can not use HAVING in the data loader flow`); if (query.orderByClause) throw new Error(`Can not USE ORDER BY in the data loader flow`); if (query.limitClause) throw new Error(`Can not use LIMIT in the data loader flow`); diff --git a/web-console/src/druid-models/workbench-query/workbench-query-part.ts b/web-console/src/druid-models/workbench-query/workbench-query-part.ts index 83a39a04c51..680a0780bd2 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query-part.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query-part.ts @@ -22,7 +22,7 @@ import Hjson from 'hjson'; import * as JSONBig from 'json-bigint-native'; import type { ColumnMetadata } from '../../utils'; -import { compact, filterMap, generate8HexId, sqlTypeFromDruid } from '../../utils'; +import { compact, filterMap, generate8HexId } from '../../utils'; import type { LastExecution } from '../execution/execution'; import { validateLastExecution } from '../execution/execution'; import { fitExternalConfigPattern } from '../external-config/external-config'; @@ -196,9 +196,9 @@ export class WorkbenchQueryPart { const { queryName, parsedQuery } = this; if (queryName && parsedQuery) { try { - return fitExternalConfigPattern(parsedQuery).signature.map(({ name, type }) => ({ - COLUMN_NAME: name, - DATA_TYPE: sqlTypeFromDruid(type), + return fitExternalConfigPattern(parsedQuery).signature.map(columnDeclaration => ({ + COLUMN_NAME: columnDeclaration.getColumnName(), + DATA_TYPE: columnDeclaration.columnType.getEffectiveType(), TABLE_NAME: queryName, TABLE_SCHEMA: 'druid', })); diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 91626668e3c..cc57096c437 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -88,11 +88,10 @@ describe('WorkbenchQuery', () => { SELECT *, VAR FROM TABLE( EXTERN( - '{"type":"http","uris":["https://static.imply.io/data/kttm/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', + '{"type":"json"}' ) - ) + ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) PARTITIONED BY ALL TIME CLUSTERED BY browser, session `; @@ -138,10 +137,9 @@ describe('WorkbenchQuery', () => { FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"}]' + '{"type":"json"}' ) - ) + ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) PARTITIONED BY HOUR CLUSTERED BY browser, session `); @@ -153,10 +151,9 @@ describe('WorkbenchQuery', () => { FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"}]' + '{"type":"json"}' ) - ) + ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) `); }); }); @@ -447,12 +444,11 @@ describe('WorkbenchQuery', () => { TIME_PARSE(pickup_datetime) AS __time, * FROM TABLE( - EXTERN( - '{"type": "local", ...}', - '{"type":"csv", ...}', - '[{ "name": "cab_type", "type": "string" }, ...]' - ) + EXTERN( + '{"type": "local", ...}', + '{"type":"csv", ...}' ) + ) EXTEND (cab_type, VARCHAR) CLUSTERED BY trip_id `; @@ -483,12 +479,11 @@ describe('WorkbenchQuery', () => { TIME_PARSE(pickup_datetime) AS __time, * FROM TABLE( - EXTERN( - '{"type": "local", ...}', - '{"type":"csv", ...}', - '[{ "name": "cab_type", "type": "string" }, ...]' - ) + EXTERN( + '{"type": "local", ...}', + '{"type":"csv", ...}' ) + ) EXTEND (cab_type, VARCHAR) CLUSTERED BY trip_id `; @@ -567,52 +562,82 @@ describe('WorkbenchQuery', () => { }); describe('#materializeHelpers', () => { - expect( - WorkbenchQuery.blank() - .changeQueryParts([ - new WorkbenchQueryPart({ - id: 'aaa', - queryName: 'kttm_data', - queryString: sane` - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' - ) - ) - `, - }), - new WorkbenchQueryPart({ - id: 'bbb', - queryName: 'country_lookup', - queryString: sane` - SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}', - '[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]' - ) - ) - `, - }), - new WorkbenchQueryPart({ - id: 'ccc', - queryName: 'x', - queryString: sane` - SELECT - os, - CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", - COUNT(DISTINCT session) AS "unique_sessions" - FROM kttm_data - LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country - GROUP BY 1, 2 - ORDER BY 3 DESC - LIMIT 10 - `, - }), - ]) - .materializeHelpers(), - ); + it('works', () => { + expect( + WorkbenchQuery.blank() + .changeQueryParts([ + new WorkbenchQueryPart({ + id: 'aaa', + queryName: 'kttm_data', + queryString: sane` + SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', + '{"type":"json"}' + ) + ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) + `, + }), + new WorkbenchQueryPart({ + id: 'bbb', + queryName: 'country_lookup', + queryString: sane` + SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', + '{"type":"tsv","findColumnsFromHeader":true}' + ) + ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) + `, + }), + new WorkbenchQueryPart({ + id: 'ccc', + queryName: 'x', + queryString: sane` + SELECT + os, + CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", + COUNT(DISTINCT session) AS "unique_sessions" + FROM kttm_data + LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country + GROUP BY 1, 2 + ORDER BY 3 DESC + LIMIT 10 + `, + }), + ]) + .materializeHelpers() + .getQueryString(), + ).toEqual(sane` + WITH + "kttm_data" AS ( + SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', + '{"type":"json"}' + ) + ) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR) + ), + "country_lookup" AS ( + SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', + '{"type":"tsv","findColumnsFromHeader":true}' + ) + ) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) + ) + ( + SELECT + os, + CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", + COUNT(DISTINCT session) AS "unique_sessions" + FROM kttm_data + LEFT JOIN country_lookup ON country_lookup.Country = kttm_data.country + GROUP BY 1, 2 + ORDER BY 3 DESC + LIMIT 10 + ) + `); + }); }); }); diff --git a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap new file mode 100644 index 00000000000..f0b47b848ca --- /dev/null +++ b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap @@ -0,0 +1,190 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`spec conversion converts index_hadoop spec (with rollup) 1`] = ` +-- This SQL query was auto generated from an ingestion spec +REPLACE INTO "newSource" OVERWRITE ALL +WITH "source" AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"s3","uris":["s3://path"]}', + '{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}' + ) +) EXTEND ("event_ts" VARCHAR, "col1" VARCHAR, "col2" VARCHAR, "col3" VARCHAR, "col4" VARCHAR, "field1" DOUBLE, "field2" DOUBLE, "field3" DOUBLE, "field4" VARCHAR, "field5" VARCHAR, "field6" BIGINT, "field7" DOUBLE)) +SELECT + TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time, + "col1", + "col2", + "col3", + "col4", + SUM("field1") AS "metric1", + MAX("field2") AS "metric2", + MIN("field3") AS "metric3", + APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4", + APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5", + SUM("field6") AS "metric6", + SUM("field7") AS "metric7" +FROM "source" +WHERE "col2" = 'xxx' +GROUP BY 1, 2, 3, 4, 5 +PARTITIONED BY HOUR +`; + +exports[`spec conversion converts index_parallel spec (with rollup) 1`] = ` +-- This SQL query was auto generated from an ingestion spec +REPLACE INTO "wikipedia_rollup" OVERWRITE ALL +WITH "source" AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}' + ) +) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "comment" VARCHAR, "isNew" VARCHAR, "isMinor" VARCHAR, "isAnonymous" VARCHAR, "user" VARCHAR, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "added" BIGINT, "commentLength" BIGINT, "delta" BIGINT, "deltaBucket" BIGINT, "deleted" BIGINT, "page" VARCHAR)) +SELECT + TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "comment", + "isNew", + "isMinor", + "isAnonymous", + "user", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName", + COUNT(*) AS "count", + SUM("added") AS "sum_added", + SUM("commentLength") AS "sum_commentLength", + SUM("delta") AS "sum_delta", + SUM("deltaBucket") AS "sum_deltaBucket", + SUM("deleted") AS "sum_deleted", + APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta" +FROM "source" +GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +PARTITIONED BY HOUR +`; + +exports[`spec conversion converts index_parallel spec (without rollup) 1`] = ` +-- This SQL query was auto generated from an ingestion spec +REPLACE INTO "wikipedia" OVERWRITE ALL +WITH "source" AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}' + ) +) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR, "event" TYPE('COMPLEX'))) +SELECT + CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + "comment", + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName", + "event" +FROM "source" +WHERE NOT ("channel" = 'xxx') +PARTITIONED BY HOUR +CLUSTERED BY "isRobot" +`; + +exports[`spec conversion converts with issue when there is a __time transform 1`] = ` +-- This SQL query was auto generated from an ingestion spec +REPLACE INTO "wikipedia" OVERWRITE ALL +WITH "source" AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}' + ) +) EXTEND ("isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR)) +SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + "comment", + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" +FROM "source" +PARTITIONED BY HOUR +CLUSTERED BY "isRobot" +`; + +exports[`spec conversion converts with issue when there is a dimension transform and strange filter 1`] = ` +-- This SQL query was auto generated from an ingestion spec +REPLACE INTO "wikipedia" OVERWRITE ALL +WITH "source" AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}' + ) +) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode" VARCHAR, "regionName" VARCHAR)) +SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" +FROM "source" +WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually +PARTITIONED BY HOUR +CLUSTERED BY "isRobot" +`; diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 2f6aa59f51e..d1fdf5b606c 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -16,10 +16,13 @@ * limitations under the License. */ -import { sane } from 'druid-query-toolkit'; - import { convertSpecToSql } from './spec-conversion'; +expect.addSnapshotSerializer({ + test: val => typeof val === 'string', + print: String, +}); + describe('spec conversion', () => { it('converts index_parallel spec (without rollup)', () => { const converted = convertSpecToSql({ @@ -96,6 +99,7 @@ describe('spec conversion', () => { 'metroCode', 'countryIsoCode', 'regionName', + { name: 'event', type: 'json' }, ], }, }, @@ -116,46 +120,7 @@ describe('spec conversion', () => { }, }); - expect(converted.queryString).toEqual(sane` - -- This SQL query was auto generated from an ingestion spec - REPLACE INTO "wikipedia" OVERWRITE ALL - WITH "source" AS (SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' - ) - )) - SELECT - CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, - "isRobot", - "channel", - "flags", - "isUnpatrolled", - "page", - "diffUrl", - "added", - "comment", - "commentLength", - "isNew", - "isMinor", - "delta", - "isAnonymous", - "user", - "deltaBucket", - "deleted", - "namespace", - "cityName", - "countryName", - "regionIsoCode", - "metroCode", - "countryIsoCode", - "regionName" - FROM "source" - WHERE NOT ("channel" = 'xxx') - PARTITIONED BY HOUR - CLUSTERED BY "isRobot" - `); + expect(converted.queryString).toMatchSnapshot(); expect(converted.queryContext).toEqual({ groupByEnableMultiValueUnnesting: false, @@ -259,45 +224,7 @@ describe('spec conversion', () => { }, }); - expect(converted.queryString).toEqual(sane` - -- This SQL query was auto generated from an ingestion spec - REPLACE INTO "wikipedia_rollup" OVERWRITE ALL - WITH "source" AS (SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"comment","type":"string"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"},{"name":"added","type":"long"},{"name":"commentLength","type":"long"},{"name":"delta","type":"long"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"page","type":"string"}]' - ) - )) - SELECT - TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time, - "isRobot", - "channel", - "flags", - "isUnpatrolled", - "comment", - "isNew", - "isMinor", - "isAnonymous", - "user", - "namespace", - "cityName", - "countryName", - "regionIsoCode", - "metroCode", - "countryIsoCode", - "regionName", - COUNT(*) AS "count", - SUM("added") AS "sum_added", - SUM("commentLength") AS "sum_commentLength", - SUM("delta") AS "sum_delta", - SUM("deltaBucket") AS "sum_deltaBucket", - SUM("deleted") AS "sum_deleted", - APPROX_COUNT_DISTINCT_DS_THETA("page") AS "page_theta" - FROM "source" - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - PARTITIONED BY HOUR - `); + expect(converted.queryString).toMatchSnapshot(); expect(converted.queryContext).toEqual({ groupByEnableMultiValueUnnesting: false, @@ -421,34 +348,7 @@ describe('spec conversion', () => { }, }); - expect(converted.queryString).toEqual(sane` - -- This SQL query was auto generated from an ingestion spec - REPLACE INTO "newSource" OVERWRITE ALL - WITH "source" AS (SELECT * FROM TABLE( - EXTERN( - '{"type":"s3","uris":["s3://path"]}', - '{"columns":["col1","col2","col3","col4","metric1","metric2","metric3","metric4","metric5","metric6","metric7"],"type":"timeAndDims"}', - '[{"name":"event_ts","type":"string"},{"name":"col1","type":"string"},{"name":"col2","type":"string"},{"name":"col3","type":"string"},{"name":"col4","type":"string"},{"name":"field1","type":"double"},{"name":"field2","type":"double"},{"name":"field3","type":"double"},{"name":"field4","type":"string"},{"name":"field5","type":"string"},{"name":"field6","type":"long"},{"name":"field7","type":"double"}]' - ) - )) - SELECT - TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts") END, 'PT1H') AS __time, - "col1", - "col2", - "col3", - "col4", - SUM("field1") AS "metric1", - MAX("field2") AS "metric2", - MIN("field3") AS "metric3", - APPROX_COUNT_DISTINCT_BUILTIN("field4") AS "metric4", - APPROX_COUNT_DISTINCT_BUILTIN("field5") AS "metric5", - SUM("field6") AS "metric6", - SUM("field7") AS "metric7" - FROM "source" - WHERE "col2" = 'xxx' - GROUP BY 1, 2, 3, 4, 5 - PARTITIONED BY HOUR - `); + expect(converted.queryString).toMatchSnapshot(); expect(converted.queryContext).toEqual({ groupByEnableMultiValueUnnesting: false, @@ -541,46 +441,7 @@ describe('spec conversion', () => { }, }); - expect(converted.queryString).toEqual(sane` - -- This SQL query was auto generated from an ingestion spec - REPLACE INTO "wikipedia" OVERWRITE ALL - WITH "source" AS (SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', - '{"type":"json"}', - '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' - ) - )) - SELECT - --:ISSUE: The spec contained transforms that could not be automatically converted. - REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted - "isRobot", - "channel", - "flags", - "isUnpatrolled", - "page", - "diffUrl", - "added", - "comment", - "commentLength", - "isNew", - "isMinor", - "delta", - "isAnonymous", - "user", - "deltaBucket", - "deleted", - "namespace", - "cityName", - "countryName", - "regionIsoCode", - "metroCode", - "countryIsoCode", - "regionName" - FROM "source" - PARTITIONED BY HOUR - CLUSTERED BY "isRobot" - `); + expect(converted.queryString).toMatchSnapshot(); }); it('converts with issue when there is a dimension transform and strange filter', () => { @@ -671,46 +532,6 @@ describe('spec conversion', () => { }, }); - expect(converted.queryString).toEqual(sane` - -- This SQL query was auto generated from an ingestion spec - REPLACE INTO "wikipedia" OVERWRITE ALL - WITH "source" AS (SELECT * FROM TABLE( - EXTERN( - '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' - ) - )) - SELECT - --:ISSUE: The spec contained transforms that could not be automatically converted. - CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, - "isRobot", - "channel", - "flags", - "isUnpatrolled", - "page", - "diffUrl", - "added", - REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted - "commentLength", - "isNew", - "isMinor", - "delta", - "isAnonymous", - "user", - "deltaBucket", - "deleted", - "namespace", - "cityName", - "countryName", - "regionIsoCode", - "metroCode", - "countryIsoCode", - "regionName" - FROM "source" - WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually - PARTITIONED BY HOUR - CLUSTERED BY "isRobot" - `); + expect(converted.queryString).toMatchSnapshot(); }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index c610866e29a..8562bd1f68f 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -16,7 +16,15 @@ * limitations under the License. */ -import { C, L, RefName, SqlExpression, T } from 'druid-query-toolkit'; +import { + C, + L, + RefName, + SqlColumnDeclaration, + SqlExpression, + SqlType, + T, +} from 'druid-query-toolkit'; import * as JSONBig from 'json-bigint-native'; import type { @@ -87,20 +95,22 @@ export function convertSpecToSql(spec: any): QueryWithContext { } dimensions = dimensions.map(inflateDimensionSpec); - let columns = dimensions.map((d: DimensionSpec) => ({ - name: d.name, - type: d.type, - })); + let columnDeclarations: SqlColumnDeclaration[] = dimensions.map((d: DimensionSpec) => + SqlColumnDeclaration.create( + d.name, + SqlType.fromNativeType(dimensionSpecTypeToNativeDataType(d.type)), + ), + ); const metricsSpec = deepGet(spec, 'spec.dataSchema.metricsSpec'); if (Array.isArray(metricsSpec)) { - columns = columns.concat( + columnDeclarations = columnDeclarations.concat( filterMap(metricsSpec, metricSpec => metricSpec.fieldName - ? { - name: metricSpec.fieldName, - type: metricSpecTypeToDataType(metricSpec.type), - } + ? SqlColumnDeclaration.create( + metricSpec.fieldName, + SqlType.fromNativeType(metricSpecTypeToNativeDataInputType(metricSpec.type)), + ) : undefined, ), ); @@ -119,42 +129,46 @@ export function convertSpecToSql(spec: any): QueryWithContext { if (timeTransform) { timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`; } else { + let timestampColumnType: SqlType; switch (format) { case 'auto': - columns.unshift({ name: timestampColumnName, type: 'string' }); + timestampColumnType = SqlType.VARCHAR; timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE TIME_PARSE(${timestampColumn}) END`; break; case 'iso': - columns.unshift({ name: timestampColumnName, type: 'string' }); + timestampColumnType = SqlType.VARCHAR; timeExpression = `TIME_PARSE(${timestampColumn})`; break; case 'posix': - columns.unshift({ name: timestampColumnName, type: 'long' }); + timestampColumnType = SqlType.BIGINT; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`; break; case 'millis': - columns.unshift({ name: timestampColumnName, type: 'long' }); + timestampColumnType = SqlType.BIGINT; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`; break; case 'micro': - columns.unshift({ name: timestampColumnName, type: 'long' }); + timestampColumnType = SqlType.BIGINT; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`; break; case 'nano': - columns.unshift({ name: timestampColumnName, type: 'long' }); + timestampColumnType = SqlType.BIGINT; timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`; break; default: - columns.unshift({ name: timestampColumnName, type: 'string' }); + timestampColumnType = SqlType.VARCHAR; timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`; break; } + columnDeclarations.unshift( + SqlColumnDeclaration.create(timestampColumnName, timestampColumnType), + ); } if (timestampSpec.missingValue) { @@ -238,10 +252,9 @@ export function convertSpecToSql(spec: any): QueryWithContext { const inputFormat = deepGet(spec, 'spec.ioConfig.inputFormat'); if (!inputFormat) throw new Error(`spec.ioConfig.inputFormat is not defined`); lines.push( - ` ${L(JSONBig.stringify(inputFormat))},`, - ` ${L(JSONBig.stringify(columns))}`, + ` ${L(JSONBig.stringify(inputFormat))}`, ` )`, - `))`, + `) EXTEND (${columnDeclarations.join(', ')}))`, ); } @@ -362,7 +375,17 @@ const QUERY_GRANULARITY_MAP: Record = { year: `TIME_FLOOR(?, 'P1Y')`, }; -function metricSpecTypeToDataType(metricSpecType: string): string { +function dimensionSpecTypeToNativeDataType(dimensionSpecType: string): string { + switch (dimensionSpecType) { + case 'json': + return 'COMPLEX'; + + default: + return dimensionSpecType; + } +} + +function metricSpecTypeToNativeDataInputType(metricSpecType: string): string { const m = /^(long|float|double|string)/.exec(String(metricSpecType)); if (m) return m[1]; diff --git a/web-console/src/utils/types.ts b/web-console/src/utils/types.ts index 4fb547ebdfd..ffb43d0115d 100644 --- a/web-console/src/utils/types.ts +++ b/web-console/src/utils/types.ts @@ -25,24 +25,6 @@ function getEffectiveColumnType(column: Column): string | undefined { return column.nativeType || column.sqlType; } -export function sqlTypeFromDruid(druidType: string): string { - druidType = druidType.toLowerCase(); - switch (druidType) { - case 'string': - return 'VARCHAR'; - - case 'long': - return 'BIGINT'; - - case 'float': - case 'double': - return druidType.toUpperCase(); - - default: - return 'COMPLEX'; - } -} - export function columnToIcon(column: Column): IconName | undefined { const effectiveType = getEffectiveColumnType(column); return effectiveType ? dataTypeToIcon(effectiveType) : undefined; diff --git a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx index e431e447e50..2edab0f3868 100644 --- a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx +++ b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx @@ -42,23 +42,25 @@ function breakdownExpression(expression: SqlExpression): Breakdown { expression = expression.getUnderlyingExpression(); let nativeType: string | undefined; - if (expression instanceof SqlFunction && expression.getEffectiveFunctionName() === 'CAST') { - const asType = String(expression.getArgAsString(1)).toUpperCase(); - switch (asType) { - case 'VARCHAR': - nativeType = 'string'; - expression = expression.getArg(0)!; - break; + if (expression instanceof SqlFunction) { + const asType = expression.getCastType(); + if (asType) { + switch (asType.value.toUpperCase()) { + case 'VARCHAR': + nativeType = 'string'; + expression = expression.getArg(0)!; + break; - case 'BIGINT': - nativeType = 'long'; - expression = expression.getArg(0)!; - break; + case 'BIGINT': + nativeType = 'long'; + expression = expression.getArg(0)!; + break; - case 'DOUBLE': - nativeType = 'double'; - expression = expression.getArg(0)!; - break; + case 'DOUBLE': + nativeType = 'double'; + expression = expression.getArg(0)!; + break; + } } } diff --git a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx index 0c9de245d50..0d6e5dc52bb 100644 --- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx +++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx @@ -451,11 +451,13 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) { }, [previewResultState]); const unusedColumns = ingestQueryPattern - ? ingestQueryPattern.mainExternalConfig.signature.filter( - ({ name }) => - !ingestQueryPattern.dimensions.some(d => d.containsColumnName(name)) && - !ingestQueryPattern.metrics?.some(m => m.containsColumnName(name)), - ) + ? ingestQueryPattern.mainExternalConfig.signature.filter(columnDeclaration => { + const columnName = columnDeclaration.getColumnName(); + return ( + !ingestQueryPattern.dimensions.some(d => d.containsColumnName(columnName)) && + !ingestQueryPattern.metrics?.some(m => m.containsColumnName(columnName)) + ); + }) : []; const timeColumn = @@ -699,23 +701,26 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) { )} {unusedColumns.length ? ( - unusedColumns.map((column, i) => ( - { - handleQueryAction(q => - q.addSelect( - C(column.name), - ingestQueryPattern.metrics - ? { insertIndex: 'last-grouping', addToGroupBy: 'end' } - : {}, - ), - ); - }} - /> - )) + unusedColumns.map((columnDeclaration, i) => { + const columnName = columnDeclaration.getColumnName(); + return ( + { + handleQueryAction(q => + q.addSelect( + C(columnName), + ingestQueryPattern.metrics + ? { insertIndex: 'last-grouping', addToGroupBy: 'end' } + : {}, + ), + ); + }} + /> + ); + }) ) : ( )} diff --git a/web-console/src/views/workbench-view/demo-queries.ts b/web-console/src/views/workbench-view/demo-queries.ts index 6d36b7261d6..d5ee1fe6cd6 100644 --- a/web-console/src/views/workbench-view/demo-queries.ts +++ b/web-console/src/views/workbench-view/demo-queries.ts @@ -54,10 +54,9 @@ SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"json"}' ) -) +) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR) PARTITIONED BY ALL TIME `, ), @@ -85,10 +84,9 @@ WITH kttm_data AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"json"}' ) -)) +) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)) SELECT FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, @@ -132,18 +130,16 @@ kttm_data AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"json"}' ) -)), +) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)), country_lookup AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}', - '[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]' + '{"type":"tsv","findColumnsFromHeader":true}' ) -)) +) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) SELECT FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, @@ -190,10 +186,9 @@ country_lookup AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}', - '[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]' + '{"type":"tsv","findColumnsFromHeader":true}' ) -)) +) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) SELECT FLOOR(TIME_PARSE("timestamp") TO MINUTE) AS __time, @@ -261,18 +256,16 @@ kttm_data AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}', - '{"type":"json"}', - '[{"name":"timestamp","type":"string"},{"name":"agent_category","type":"string"},{"name":"agent_type","type":"string"},{"name":"browser","type":"string"},{"name":"browser_version","type":"string"},{"name":"city","type":"string"},{"name":"continent","type":"string"},{"name":"country","type":"string"},{"name":"version","type":"string"},{"name":"event_type","type":"string"},{"name":"event_subtype","type":"string"},{"name":"loaded_image","type":"string"},{"name":"adblock_list","type":"string"},{"name":"forwarded_for","type":"string"},{"name":"language","type":"string"},{"name":"number","type":"long"},{"name":"os","type":"string"},{"name":"path","type":"string"},{"name":"platform","type":"string"},{"name":"referrer","type":"string"},{"name":"referrer_host","type":"string"},{"name":"region","type":"string"},{"name":"remote_address","type":"string"},{"name":"screen","type":"string"},{"name":"session","type":"string"},{"name":"session_length","type":"long"},{"name":"timezone","type":"string"},{"name":"timezone_offset","type":"long"},{"name":"window","type":"string"}]' + '{"type":"json"}' ) -)), +) EXTEND ("timestamp" VARCHAR, "agent_category" VARCHAR, "agent_type" VARCHAR, "browser" VARCHAR, "browser_version" VARCHAR, "city" VARCHAR, "continent" VARCHAR, "country" VARCHAR, "version" VARCHAR, "event_type" VARCHAR, "event_subtype" VARCHAR, "loaded_image" VARCHAR, "adblock_list" VARCHAR, "forwarded_for" VARCHAR, "language" VARCHAR, "number" VARCHAR, "os" VARCHAR, "path" VARCHAR, "platform" VARCHAR, "referrer" VARCHAR, "referrer_host" VARCHAR, "region" VARCHAR, "remote_address" VARCHAR, "screen" VARCHAR, "session" VARCHAR, "session_length" BIGINT, "timezone" VARCHAR, "timezone_offset" VARCHAR, "window" VARCHAR)), country_lookup AS ( SELECT * FROM TABLE( EXTERN( '{"type":"http","uris":["https://static.imply.io/example-data/lookup/countries.tsv"]}', - '{"type":"tsv","findColumnsFromHeader":true}', - '[{"name":"Country","type":"string"},{"name":"Capital","type":"string"},{"name":"ISO3","type":"string"},{"name":"ISO2","type":"string"}]' + '{"type":"tsv","findColumnsFromHeader":true}' ) -)) +) EXTEND ("Country" VARCHAR, "Capital" VARCHAR, "ISO3" VARCHAR, "ISO2" VARCHAR)) SELECT os, CONCAT(country, ' (', country_lookup.ISO3, ')') AS "country", diff --git a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap index eb314213902..77762a71575 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap @@ -38,7 +38,7 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` >

- General info for query-955fa5a2-0ae6-4912-bc39-cc9fc012de5a ingesting into "kttm_simple" + General info for query-614dc100-a4b9-40a3-95ce-1227fa7ea765 ingesting into "kttm-blank-lines"

: - Too many warnings of type CannotParseExternalData generated (max = 10) + Too many warnings of type CannotParseExternalData generated (max = 2)

Failed task ID: (on host: diff --git a/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap index c824afb1fb0..46e02c90454 100644 --- a/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap @@ -158,7 +158,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` Array [ Object { "definition": Object { - "id": "b94caff0-f693-47ed-a242-ccf837550383_0", + "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0", "input": Array [ Object { "inputFormat": Object { @@ -186,6 +186,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` "type": "external", }, ], + "maxInputBytesPerWorker": 10737418240, "maxWorkerCount": 1, "processor": Object { "query": Object { @@ -198,10 +199,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` "finalize": false, "finalizeAggregations": false, "groupByEnableMultiValueUnnesting": false, - "queryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "maxNumTasks": 2, + "queryId": "b55f3432-7810-4529-80ed-780a926a6f03", "scanSignature": "[{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"v0\\",\\"type\\":\\"LONG\\"}]", "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}", - "sqlQueryId": "09af0c1e-1c0f-4539-917c-b0458849d0d9", + "sqlQueryId": "b55f3432-7810-4529-80ed-780a926a6f03", "sqlReplaceTimeChunks": "all", }, "dataSource": Object { @@ -237,6 +239,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` "columns": Array [ Object { "columnName": "__boost", + "order": "ASCENDING", }, ], }, @@ -258,23 +261,24 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` }, ], }, - "duration": 4887, + "duration": 24236, "partitionCount": 1, "phase": "FINISHED", "sort": true, "stageNumber": 0, - "startTime": "2022-10-31T16:11:15.380Z", + "startTime": "2023-03-27T22:17:02.792Z", "workerCount": 1, }, Object { "definition": Object { - "id": "b94caff0-f693-47ed-a242-ccf837550383_1", + "id": "8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1", "input": Array [ Object { "stage": 0, "type": "stage", }, ], + "maxInputBytesPerWorker": 10737418240, "maxWorkerCount": 1, "processor": Object { "columnMappings": Array [ @@ -302,6 +306,7 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` }, ], "includeAllDimensions": false, + "useSchemaDiscovery": false, }, "granularitySpec": Object { "intervals": Array [ @@ -333,11 +338,11 @@ exports[`ExecutionStagesPane matches snapshot 1`] = ` }, "signature": Array [], }, - "duration": 1908, + "duration": 4276, "partitionCount": 1, "phase": "FINISHED", "stageNumber": 1, - "startTime": "2022-10-31T16:11:20.264Z", + "startTime": "2023-03-27T22:17:26.978Z", "workerCount": 1, }, ] diff --git a/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap b/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap index df513e4a6d8..0533f055011 100644 --- a/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/ingest-success-pane/__snapshots__/ingest-success-pane.spec.tsx.snap @@ -8,7 +8,7 @@ exports[`IngestSuccessPane matches snapshot 1`] = ` 465,346 rows inserted into "kttm_simple".

- Insert query took 0:00:09. + Insert query took 0:00:28. {}; export interface InputFormatAndMore { inputFormat: InputFormat; - signature: SignatureColumn[]; + signature: SqlColumnDeclaration[]; isArrays: boolean[]; timeExpression: SqlExpression | undefined; } @@ -121,7 +121,7 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF let possibleTimeExpression: PossibleTimeExpression | undefined; if (previewData) { possibleTimeExpression = filterMap(previewData.header, column => { - const values = previewData.rows.map(row => row.input[column]); + const values = filterMap(previewData.rows, row => row.input?.[column]); const possibleDruidFormat = possibleDruidFormatForValues(values); if (!possibleDruidFormat) return; @@ -139,14 +139,18 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF previewData && AutoForm.isValidModel(inputFormat, INPUT_FORMAT_FIELDS) ? { inputFormat, - signature: previewData.header.map(name => ({ - name, - type: guessColumnTypeFromHeaderAndRows( - previewData, + signature: previewData.header.map(name => + SqlColumnDeclaration.create( name, - inputFormatOutputsNumericStrings(inputFormat), + SqlType.fromNativeType( + guessColumnTypeFromHeaderAndRows( + previewData, + name, + inputFormatOutputsNumericStrings(inputFormat), + ), + ), ), - })), + ), isArrays: previewData.header.map(name => guessIsArrayFromHeaderAndRows(previewData, name), ), diff --git a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx index c4eaf8d0afa..4a8ba96f260 100644 --- a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx +++ b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx @@ -29,6 +29,7 @@ import { import { IconNames } from '@blueprintjs/icons'; import classNames from 'classnames'; import type { QueryResult } from 'druid-query-toolkit'; +import { SqlColumnDeclaration } from 'druid-query-toolkit'; import React, { useEffect, useState } from 'react'; import { AutoForm, ExternalLink } from '../../../components'; @@ -138,7 +139,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS listDelimiter: BOGUS_LIST_DELIMITER, columns: ['raw'], }, - signature: [{ name: 'raw', type: 'string' }], + signature: [SqlColumnDeclaration.create('raw', 'VARCHAR')], }); const result = extractResult( diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index a5cd837f87f..743e44174f8 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -165,10 +165,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result // Casts if (selectExpression) { const underlyingExpression = selectExpression.getUnderlyingExpression(); - if ( - underlyingExpression instanceof SqlFunction && - underlyingExpression.getEffectiveFunctionName() === 'CAST' - ) { + if (underlyingExpression instanceof SqlFunction && underlyingExpression.getCastType()) { menuItems.push( + + {['broadcast', 'sortMerge'].map(o => ( + + changeQueryContext(deepSet(queryContext, 'sqlJoinAlgorithm', o)) + } + /> + ))} +