From d9c9aef3d18a394276238d741a3241c12fb34b5b Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 23 Oct 2019 16:50:08 -0700 Subject: [PATCH] Druid Doctor (#8672) * adding Druid doctor * better meesage * feedback changes * add icons * feedback fixes * spelling * add file.encoding check * feedback changes --- web-console/package-lock.json | 107 +---- web-console/package.json | 4 +- .../__snapshots__/header-bar.spec.tsx.snap | 9 + .../src/components/header-bar/header-bar.tsx | 8 + .../__snapshots__/about-dialog.spec.tsx.snap | 2 +- .../src/dialogs/about-dialog/about-dialog.tsx | 9 +- .../__snapshots__/doctor-dialog.spec.tsx.snap | 118 +++++ .../dialogs/doctor-dialog/doctor-checks.tsx | 423 ++++++++++++++++++ .../dialogs/doctor-dialog/doctor-dialog.scss | 33 ++ .../doctor-dialog/doctor-dialog.spec.tsx | 31 ++ .../dialogs/doctor-dialog/doctor-dialog.tsx | 200 +++++++++ web-console/src/utils/general.tsx | 6 + web-console/src/utils/ingestion-spec.tsx | 2 +- web-console/src/utils/sampler.ts | 5 +- 14 files changed, 867 insertions(+), 90 deletions(-) create mode 100644 web-console/src/dialogs/doctor-dialog/__snapshots__/doctor-dialog.spec.tsx.snap create mode 100644 web-console/src/dialogs/doctor-dialog/doctor-checks.tsx create mode 100644 web-console/src/dialogs/doctor-dialog/doctor-dialog.scss create mode 100644 web-console/src/dialogs/doctor-dialog/doctor-dialog.spec.tsx create mode 100644 web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 3447dbc1059..f6a80962693 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -839,16 +839,17 @@ } }, "@blueprintjs/core": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.18.0.tgz", - "integrity": "sha512-dr3A6uhpAAWmf5muY6PFQp5EgEzinRLZa/TGQMA05q1P2xrOn/LYlsqJWJBUJ3j9tmh1RP8VPeu1HmosQb3J7w==", + "version": "3.19.1", + "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.19.1.tgz", + "integrity": "sha512-O+p/Jbu9kyrTE4Yy5phd7Xp9NJoiHCQ9ycO+QSR2fiuhwnVMO1xjY269QXxylzEswgPyjhST1euqHpyQQ7qnUw==", "requires": { - "@blueprintjs/icons": "^3.10.0", + "@blueprintjs/icons": "^3.11.0", "@types/dom4": "^2.0.1", "classnames": "^2.2", "dom4": "^2.1.5", "normalize.css": "^8.0.1", "popper.js": "^1.15.0", + "react-lifecycles-compat": "^3.0.4", "react-popper": "^1.3.3", "react-transition-group": "^2.9.0", "resize-observer-polyfill": "^1.5.1", @@ -863,9 +864,9 @@ } }, "@blueprintjs/icons": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.10.0.tgz", - "integrity": "sha512-lyAUpkr3qEStPcJpMnxRKuVAPvaRNSce1ySPbkE58zPmD4WBya2gNrWex41xoqRYM0GsiBSwH9CnpY8t6fZKUA==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.11.0.tgz", + "integrity": "sha512-HGS652gFc057t9cr8NyuWFyZ1gcSqG3uuexpzhZm81W35hGfh9vdC9GR+mbHJNawAuKXtu+xw4VWWkv1UGQ0Vg==", "requires": { "classnames": "^2.2", "tslib": "~1.9.0" @@ -2243,7 +2244,8 @@ "asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=", + "dev": true }, "asn1": { "version": "0.2.4", @@ -3556,12 +3558,12 @@ } }, "create-react-context": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/create-react-context/-/create-react-context-0.2.2.tgz", - "integrity": "sha512-KkpaLARMhsTsgp0d2NA/R94F/eDLbhXERdIq3LvX2biCAXcDvHYoOqHfWCHf1+OLj+HKBotLG3KqaOOf+C1C+A==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/create-react-context/-/create-react-context-0.3.0.tgz", + "integrity": "sha512-dNldIoSuNSvlTJ7slIKC/ZFGKexBMBrrcc+TTe1NdmROnaASuLPvqpwj9v4XS4uXZ8+YPu0sNmShX2rXI5LNsw==", "requires": { - "fbjs": "^0.8.0", - "gud": "^1.0.0" + "gud": "^1.0.0", + "warning": "^4.0.3" } }, "cross-spawn": { @@ -4574,14 +4576,6 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", "dev": true }, - "encoding": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", - "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=", - "requires": { - "iconv-lite": "~0.4.13" - } - }, "end-of-stream": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", @@ -5176,27 +5170,6 @@ "bser": "^2.0.0" } }, - "fbjs": { - "version": "0.8.17", - "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.17.tgz", - "integrity": "sha1-xNWY6taUkRJlPWWIsBpc3Nn5D90=", - "requires": { - "core-js": "^1.0.0", - "isomorphic-fetch": "^2.1.1", - "loose-envify": "^1.0.0", - "object-assign": "^4.1.0", - "promise": "^7.1.1", - "setimmediate": "^1.0.5", - "ua-parser-js": "^0.7.18" - }, - "dependencies": { - "core-js": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", - "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY=" - } - } - }, "figgy-pudding": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz", @@ -7245,7 +7218,8 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true }, "is-string": { "version": "1.0.4", @@ -7321,15 +7295,6 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, - "isomorphic-fetch": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", - "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", - "requires": { - "node-fetch": "^1.0.1", - "whatwg-fetch": ">=0.10.0" - } - }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -8970,15 +8935,6 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, - "node-fetch": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", - "requires": { - "encoding": "^0.1.11", - "is-stream": "^1.0.1" - } - }, "node-forge": { "version": "0.7.5", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.5.tgz", @@ -10691,14 +10647,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, - "promise": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", - "requires": { - "asap": "~2.0.3" - } - }, "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", @@ -10965,12 +10913,12 @@ "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==" }, "react-popper": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/react-popper/-/react-popper-1.3.3.tgz", - "integrity": "sha512-ynMZBPkXONPc5K4P5yFWgZx5JGAUIP3pGGLNs58cfAPgK67olx7fmLp+AdpZ0+GoQ+ieFDa/z4cdV6u7sioH6w==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/react-popper/-/react-popper-1.3.4.tgz", + "integrity": "sha512-9AcQB29V+WrBKk6X7p0eojd1f25/oJajVdMZkywIoAV6Ag7hzE1Mhyeup2Q1QnvFRtGQFQvtqfhlEoDAPfKAVA==", "requires": { "@babel/runtime": "^7.1.2", - "create-react-context": "<=0.2.2", + "create-react-context": "^0.3.0", "popper.js": "^1.14.4", "prop-types": "^15.6.1", "typed-styles": "^0.0.7", @@ -12099,7 +12047,8 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", + "dev": true }, "setprototypeof": { "version": "1.1.1", @@ -13931,11 +13880,6 @@ "integrity": "sha512-ACzBtm/PhXBDId6a6sDJfroT2pOWt/oOnk4/dElG5G33ZL776N3Y6/6bKZJBFpd+b05F3Ct9qDjMeJmRWtE2/g==", "dev": true }, - "ua-parser-js": { - "version": "0.7.20", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz", - "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw==" - }, "uglify-js": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", @@ -14767,11 +14711,6 @@ "iconv-lite": "0.4.24" } }, - "whatwg-fetch": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz", - "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==" - }, "whatwg-mimetype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", diff --git a/web-console/package.json b/web-console/package.json index 5ba4686724f..63ba12f80bd 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -53,8 +53,8 @@ "start": "webpack-dev-server --hot --open" }, "dependencies": { - "@blueprintjs/core": "^3.18.0", - "@blueprintjs/icons": "^3.10.0", + "@blueprintjs/core": "^3.19.1", + "@blueprintjs/icons": "^3.11.0", "axios": "^0.19.0", "brace": "^0.11.1", "classnames": "^2.2.6", diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index 6d7a6e0a197..f055cc5a2d1 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -94,6 +94,15 @@ exports[`header bar matches snapshot 1`] = ` captureDismiss={false} content={ + + setDoctorDialogOpen(true)} + /> {aboutDialogOpen && setAboutDialogOpen(false)} />} + {doctorDialogOpen && setDoctorDialogOpen(false)} />} {coordinatorDynamicConfigDialogOpen && ( setCoordinatorDynamicConfigDialogOpen(false)} diff --git a/web-console/src/dialogs/about-dialog/__snapshots__/about-dialog.spec.tsx.snap b/web-console/src/dialogs/about-dialog/__snapshots__/about-dialog.spec.tsx.snap index e879e85c0df..6b5f30e47db 100644 --- a/web-console/src/dialogs/about-dialog/__snapshots__/about-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/about-dialog/__snapshots__/about-dialog.spec.tsx.snap @@ -16,7 +16,7 @@ exports[`about dialog matches snapshot 1`] = ` tabindex="0" >
+

diff --git a/web-console/src/dialogs/doctor-dialog/__snapshots__/doctor-dialog.spec.tsx.snap b/web-console/src/dialogs/doctor-dialog/__snapshots__/doctor-dialog.spec.tsx.snap new file mode 100644 index 00000000000..f55ed987201 --- /dev/null +++ b/web-console/src/dialogs/doctor-dialog/__snapshots__/doctor-dialog.spec.tsx.snap @@ -0,0 +1,118 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`doctor dialog matches snapshot 1`] = ` +

+
+
+
+
+
+ + + + pulse + + + + +

+ Druid Doctor +

+ +
+
+
+
+ Automated checks to troubleshoot issues with the cluster. +
+ +
+
+
+
+ +
+
+
+
+
+
+`; diff --git a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx new file mode 100644 index 00000000000..e8b90da8a27 --- /dev/null +++ b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import axios from 'axios'; + +import { pluralIfNeeded, queryDruidSql } from '../../utils'; +import { deepGet } from '../../utils/object-change'; +import { postToSampler } from '../../utils/sampler'; + +export interface CheckControls { + addSuggestion: (message: string) => void; + addIssue: (message: string) => void; + terminateChecks: () => void; +} + +export interface DoctorCheck { + name: string; + check: (controls: CheckControls) => Promise; +} + +const RUNTIME_PROPERTIES_ALL_NODES_MUST_AGREE_ON: string[] = [ + 'user.timezone', + 'druid.zk.service.host', +]; + +// In the future (when we can query other nodes) is will also be cool to check: +// 'druid.storage.type' <=> historicals, overlords, mm +// 'druid.indexer.logs.type' <=> overlord, mm, + peons + +const RUNTIME_PROPERTIES_MASTER_NODES_SHOULD_AGREE_ON: string[] = [ + 'druid.metadata.storage.type', // overlord + coordinator + 'druid.metadata.storage.connector.connectURI', +]; + +export const DOCTOR_CHECKS: DoctorCheck[] = [ + // ------------------------------------- + // Self (router) checks + // ------------------------------------- + { + name: 'Verify own status', + check: async controls => { + // Make sure that the router responds to /status and gives some valid info back + let status: any; + try { + status = (await axios.get(`/status`)).data; + } catch (e) { + controls.addIssue( + `Did not get a /status response from the Router node. Try confirming that it is running and accessible. Got: ${e.message}`, + ); + controls.terminateChecks(); + return; + } + + if (typeof status.version !== 'string') { + controls.addIssue('Could not get a valid /status response from the Router.'); + } + }, + }, + { + name: 'Verify own runtime properties', + check: async controls => { + // Make sure that everything in /status/properties is above board + let properties: Record; + try { + properties = (await axios.get(`/status/properties`)).data; + } catch (e) { + controls.addIssue( + `Did not get a /status/properties response from the Router. Message: ${e.message}`, + ); + return; + } + + // Check that the management proxy is on, it really should be for someone to access the console in the first place but everything could happen + if (properties['druid.router.managementProxy.enabled'] !== 'true') { + controls.addIssue( + `The Router's "druid.router.managementProxy.enabled" is not reported as "true". This means that the Coordinator and Overlord will not be accessible from the Router (and this console).`, + ); + } + + // Check that the underlying Java is Java 8 the only officially supported Java version at the moment. + if ( + properties['java.specification.version'] && + properties['java.specification.version'] !== '1.8' + ) { + controls.addSuggestion( + `It looks like are running Java ${properties['java.runtime.version']}. Druid only officially supports Java 1.8.x`, + ); + } + + // Check "file.encoding" + if (properties['file.encoding'] && properties['file.encoding'] !== 'UTF-8') { + controls.addSuggestion( + `It looks like "file.encoding" is set to ${properties['file.encoding']}, it is recommended to set this to "UTF-8"`, + ); + } + + // Check "user.timezone" + if (properties['user.timezone'] && properties['user.timezone'] !== 'UTC') { + controls.addSuggestion( + `It looks like "user.timezone" is set to ${properties['user.timezone']}, it is recommended to set this to "UTC"`, + ); + } + }, + }, + + // ------------------------------------- + // Coordinator and Overlord + // ------------------------------------- + { + name: 'Verify the Coordinator and Overlord status', + check: async controls => { + // Make sure that everything in Coordinator's /status is good + let myStatus: any; + try { + myStatus = (await axios.get(`/status`)).data; + } catch { + return; + } + + let coordinatorStatus: any; + try { + coordinatorStatus = (await axios.get(`/proxy/coordinator/status`)).data; + } catch (e) { + controls.addIssue( + 'Did not get a /status response from the Coordinator node. Try confirming that it is running and accessible.', + ); + return; + } + + let overlordStatus: any; + try { + overlordStatus = (await axios.get(`/proxy/overlord/status`)).data; + } catch (e) { + controls.addIssue( + 'Did not get a /status response from the Overlord node. Try confirming that it is running and accessible.', + ); + return; + } + + if (myStatus.version !== coordinatorStatus.version) { + controls.addSuggestion( + `It looks like the Router and Coordinator nodes are on different versions of Druid. This may indicate a problem if you are not in the middle of a rolling upgrade.`, + ); + } + + if (myStatus.version !== overlordStatus.version) { + controls.addSuggestion( + `It looks like the Router and Overlord nodes are on different versions of Druid. This may indicate a problem if you are not in the middle of a rolling upgrade.`, + ); + } + }, + }, + { + name: 'Verify the Coordinator and Overlord runtime properties', + check: async controls => { + // Make sure that everything in coordinator and overlord /status/properties is good and matches where needed + let myProperties: Record; + try { + myProperties = (await axios.get(`/status/properties`)).data; + } catch { + return; + } + + let coordinatorProperties: Record; + try { + coordinatorProperties = (await axios.get(`/proxy/coordinator/status/properties`)).data; + } catch (e) { + controls.addIssue( + 'Did not get a /status response from the coordinator. Try confirming that it is running and accessible.', + ); + return; + } + + let overlordProperties: Record; + try { + overlordProperties = (await axios.get(`/proxy/overlord/status/properties`)).data; + } catch (e) { + controls.addIssue( + 'Did not get a /status response from the overlord. Try confirming that it is running and accessible.', + ); + return; + } + + for (const prop of RUNTIME_PROPERTIES_ALL_NODES_MUST_AGREE_ON) { + if (myProperties[prop] !== coordinatorProperties[prop]) { + controls.addIssue( + `The Router and Coordinator do not agree on the "${prop}" runtime property ("${myProperties[prop]}" vs "${coordinatorProperties[prop]}")`, + ); + } + if (myProperties[prop] !== overlordProperties[prop]) { + controls.addIssue( + `The Router and Overlord do not agree on the "${prop}" runtime property ("${myProperties[prop]}" vs "${overlordProperties[prop]}")`, + ); + } + } + + for (const prop of RUNTIME_PROPERTIES_MASTER_NODES_SHOULD_AGREE_ON) { + if (coordinatorProperties[prop] !== overlordProperties[prop]) { + controls.addSuggestion( + `The Coordinator and Overlord do not agree on the "${prop}" runtime property ("${coordinatorProperties[prop]}" vs "${overlordProperties[prop]}")`, + ); + } + } + }, + }, + + // ------------------------------------- + // Check sampler + // ------------------------------------- + { + name: 'Verify that the sampler works', + check: async controls => { + // Make sure that everything in Coordinator's /status is good + let testSampledData: any; + try { + testSampledData = await postToSampler( + { + type: 'index', + spec: { + type: 'index', + ioConfig: { type: 'index', firehose: { type: 'inline', data: '{"test":"Data"}' } }, + dataSchema: { + dataSource: 'sample', + parser: { + type: 'string', + parseSpec: { + format: 'json', + timestampSpec: { + column: '!!!_no_such_column_!!!', + missingValue: '2010-01-01T00:00:00Z', + }, + dimensionsSpec: { dimensions: ['test'] }, + }, + }, + transformSpec: {}, + metricsSpec: [], + granularitySpec: { queryGranularity: 'NONE' }, + }, + }, + samplerConfig: { + numRows: 50, + timeoutMs: 1000, + }, + }, + 'doctor', + ); + } catch { + controls.addIssue(`Could not use the sampler.`); + return; + } + + if (deepGet(testSampledData, 'data.0.parsed.test') !== 'Data') { + controls.addIssue(`Sampler returned incorrect data.`); + } + }, + }, + + // ------------------------------------- + // Check SQL + // ------------------------------------- + { + name: 'Verify that SQL works', + check: async controls => { + // Make sure that we can run the simplest query + let sqlResult: any[]; + try { + sqlResult = await queryDruidSql({ query: `SELECT 1 + 1 AS "two"` }); + } catch (e) { + controls.addIssue( + `Could not query SQL ensure that "druid.sql.enable" is set to "true" and that there is a Broker node running. Got: ${e.message}`, + ); + controls.terminateChecks(); + return; + } + + if (sqlResult.length !== 1 || sqlResult[0]['two'] !== 2) { + controls.addIssue(`Got incorrect results from a basic SQL query.`); + } + }, + }, + { + name: 'Verify that there are historical nodes', + check: async controls => { + // Make sure that there are broker and historical nodes reported from sys.servers + let sqlResult: any[]; + try { + sqlResult = await queryDruidSql({ + query: `SELECT + COUNT(*) AS "historicals" +FROM sys.servers +WHERE "server_type" = 'historical'`, + }); + } catch (e) { + controls.addIssue(`Could not run a sys.servers query. Got: ${e.message}`); + return; + } + + if (sqlResult.length === 1 && sqlResult[0]['historicals'] === 0) { + controls.addIssue(`There do not appear to be any historical nodes.`); + } + }, + }, + { + name: 'Verify that the historicals are not overfilled', + check: async controls => { + // Make sure that no nodes are reported that are over 95% capacity + let sqlResult: any[]; + try { + sqlResult = await queryDruidSql({ + query: `SELECT + "server", + "curr_size" * 1.0 / "max_size" AS "fill" +FROM sys.servers +WHERE "server_type" = 'historical' AND "curr_size" * 1.0 / "max_size" > 0.9 +ORDER BY "server" DESC`, + }); + } catch (e) { + controls.addIssue(`Could not run a sys.servers query. Got: ${e.message}`); + return; + } + + function formatPercent(server: any): string { + return (server['fill'] * 100).toFixed(2); + } + + for (const server of sqlResult) { + if (server['fill'] > 0.95) { + controls.addIssue( + `Server "${server['server']}" appears to be over 95% full (is ${formatPercent( + server, + )}%). Increase capacity.`, + ); + } else { + controls.addSuggestion( + `Server "${server['server']}" appears to be over 90% full (is ${formatPercent( + server, + )}%)`, + ); + } + } + }, + }, + { + name: 'Look for time chunks that could benefit from compaction', + check: async controls => { + // Check for any time chunks where there is more than 1 segment and avg segment size is less than 100MB + const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + let sqlResult: any[]; + try { + sqlResult = await queryDruidSql({ + query: `SELECT + "datasource", + COUNT(*) AS "num_bad_time_chunks" +FROM ( + SELECT + "datasource", "start", "end", + AVG("size") AS "avg_segment_size_in_time_chunk", + SUM("size") AS "total_size", + COUNT(*) AS "num_segments" + FROM sys.segments + WHERE is_published = 1 AND "start" < '${dayAgo}' + GROUP BY 1, 2, 3 + HAVING "num_segments" > 1 AND "total_size" > 1 AND "avg_segment_size_in_time_chunk" < 100000000 +) +GROUP BY 1 +ORDER BY "num_bad_time_chunks"`, + }); + } catch (e) { + return; + } + + if (sqlResult.length) { + // Grab the auto-compaction definitions and ignore dataSources that already have auto-compaction + let compactionResult: any; + try { + compactionResult = (await axios.get('/druid/coordinator/v1/config/compaction')).data; + } catch (e) { + controls.addIssue(`Could not get compaction config. Something is wrong.`); + return; + } + + if (!compactionResult.compactionConfigs) return; + + if (!Array.isArray(compactionResult.compactionConfigs)) { + controls.addIssue(`Got invalid value from compaction config. Something is wrong.`); + return; + } + + const dataSourcesWithCompaction = compactionResult.compactionConfigs.map( + (d: any) => d.dataSource, + ); + + sqlResult = sqlResult.filter(d => !dataSourcesWithCompaction.includes(d['datasource'])); + + for (const datasource of sqlResult) { + controls.addSuggestion( + `Datasource "${ + datasource['datasource'] + }" could benefit from auto-compaction as it has ${pluralIfNeeded( + datasource['num_bad_time_chunks'], + 'time chunk', + )} that have multiple small segments that could be compacted.`, + ); + } + } + }, + }, +]; diff --git a/web-console/src/dialogs/doctor-dialog/doctor-dialog.scss b/web-console/src/dialogs/doctor-dialog/doctor-dialog.scss new file mode 100644 index 00000000000..7efca3a5d14 --- /dev/null +++ b/web-console/src/dialogs/doctor-dialog/doctor-dialog.scss @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.doctor-dialog { + &.bp3-dialog { + margin-top: 5vh; + top: 5%; + } + + .bp3-dialog-body { + height: 70vh; + overflow: scroll; + } + + .diagnosis { + margin-bottom: 10px; + } +} diff --git a/web-console/src/dialogs/doctor-dialog/doctor-dialog.spec.tsx b/web-console/src/dialogs/doctor-dialog/doctor-dialog.spec.tsx new file mode 100644 index 00000000000..744e9401ce6 --- /dev/null +++ b/web-console/src/dialogs/doctor-dialog/doctor-dialog.spec.tsx @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; + +import { DoctorDialog } from './doctor-dialog'; + +describe('doctor dialog', () => { + it('matches snapshot', () => { + const doctorDialog = {}} />; + + render(doctorDialog); + expect(document.body.lastChild).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx b/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx new file mode 100644 index 00000000000..3b262976382 --- /dev/null +++ b/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Callout, Classes, Dialog, Intent } from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; +import React from 'react'; + +import { delay, pluralIfNeeded } from '../../utils'; + +import { DOCTOR_CHECKS } from './doctor-checks'; + +import './doctor-dialog.scss'; + +interface Diagnosis { + type: 'suggestion' | 'issue'; + check: string; + message: string; +} + +export interface DoctorDialogProps { + onClose: () => void; +} + +export interface DoctorDialogState { + currentCheckIndex?: number; + diagnoses?: Diagnosis[]; + earlyTermination?: string; +} + +export class DoctorDialog extends React.PureComponent { + private mounted = false; + + constructor(props: DoctorDialogProps, context: any) { + super(props, context); + this.state = {}; + } + + componentDidMount(): void { + this.mounted = true; + } + + componentWillUnmount(): void { + this.mounted = false; + } + + async doChecks() { + this.setState({ currentCheckIndex: 0, diagnoses: [] }); + + const addToDiagnoses = (diagnosis: Diagnosis) => { + if (!this.mounted) return; + this.setState(oldState => ({ + diagnoses: (oldState.diagnoses || []).concat(diagnosis), + })); + }; + + for (let i = 0; i < DOCTOR_CHECKS.length; i++) { + if (!this.mounted) return; + this.setState({ currentCheckIndex: i }); + const check = DOCTOR_CHECKS[i]; + let terminateChecks = false; + + try { + await Promise.all([ + await delay(450), // Make sure that a test takes at least this long so that the user can read the test name in the GUI, + check.check({ + addSuggestion: (message: string) => { + addToDiagnoses({ + type: 'suggestion', + check: check.name, + message, + }); + }, + addIssue: (message: string) => { + addToDiagnoses({ + type: 'issue', + check: check.name, + message, + }); + }, + terminateChecks: () => { + if (!this.mounted) return; + this.setState({ + earlyTermination: `The check "${check.name}" early terminated the check suite as it has encountered a condition that would make the rest of the tests meaningless.`, + }); + terminateChecks = true; + }, + }), + ]); + } catch (e) { + addToDiagnoses({ + type: 'issue', + check: check.name, + message: `The check "${check.name}" encountered an unhandled exception. Please report this issue. Message: ${e.message}`, + }); + } + + if (terminateChecks) break; + } + + if (!this.mounted) return; + this.setState({ currentCheckIndex: undefined }); + } + + renderContent() { + const { diagnoses, currentCheckIndex, earlyTermination } = this.state; + + if (diagnoses) { + let note: string; + if (typeof currentCheckIndex === 'number') { + note = `Running check ${currentCheckIndex + 1}/${DOCTOR_CHECKS.length}: ${ + DOCTOR_CHECKS[currentCheckIndex].name + }`; + } else if (earlyTermination) { + note = `Checks stopped abruptly`; + } else { + note = `All ${pluralIfNeeded(DOCTOR_CHECKS.length, 'check')} completed`; + } + + return ( + <> + {note} + {diagnoses.map((diagnosis, i) => { + return ( + + {diagnosis.message} + + ); + })} + {earlyTermination && ( + + {earlyTermination} + + )} + {!earlyTermination && currentCheckIndex == null && diagnoses.length === 0 && ( + + No issues detected + + )} + + ); + } else { + return ( +
+ + Automated checks to troubleshoot issues with the cluster. + +
+ ); + } + } + + render(): JSX.Element { + const { onClose } = this.props; + + return ( + +
{this.renderContent()}
+
+
+ +
+
+ + ); + } +} diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 6444392c50a..9c125fa7d19 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -338,3 +338,9 @@ export function copyAndAlert(copyString: string, alertMessage: string): void { intent: Intent.SUCCESS, }); } + +export function delay(ms: number) { + return new Promise(resolve => { + setTimeout(resolve, ms); + }); +} diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx index 12f29bd862b..7fa4e04a573 100644 --- a/web-console/src/utils/ingestion-spec.tsx +++ b/web-console/src/utils/ingestion-spec.tsx @@ -547,7 +547,7 @@ export function getFlattenFieldFormFields() { } export interface TransformSpec { - transforms: Transform[]; + transforms?: Transform[]; filter?: any; } diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index 079865548d4..83c0401a81c 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -151,7 +151,10 @@ export async function getOverlordModules(): Promise { return statusResp.data.modules.map((m: any) => m.artifact); } -async function postToSampler(sampleSpec: SampleSpec, forStr: string): Promise { +export async function postToSampler( + sampleSpec: SampleSpec, + forStr: string, +): Promise { let sampleResp: any; try { sampleResp = await axios.post(`${SAMPLER_URL}?for=${forStr}`, sampleSpec);