From aee2f2e24f4230156a46d8f570a8d4a37b5f0385 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 20 Jul 2021 17:17:19 -0700 Subject: [PATCH] Web console: better handle BigInt math (#11450) * better handle BigInt math * correctly brace bigint * feedback fixes and tests --- web-console/src/utils/general.tsx | 41 ++++--- .../views/datasource-view/datasource-view.tsx | 83 +++++++------ .../query-view/query-output/query-output.tsx | 11 +- .../src/views/segments-view/segments-view.tsx | 3 +- .../__snapshots__/services-view.spec.tsx.snap | 39 ++++++- .../services-view/services-view.spec.tsx | 70 ++++++++++- .../src/views/services-view/services-view.tsx | 110 ++++++++++-------- 7 files changed, 245 insertions(+), 112 deletions(-) mode change 100755 => 100644 web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index e16e337239b..1b0aa4fcc00 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -33,6 +33,12 @@ import { AppToaster } from '../singletons'; export const EMPTY_OBJECT: any = {}; export const EMPTY_ARRAY: any[] = []; +export type NumberLike = number | BigInt; + +export function isNumberLikeNaN(x: NumberLike): boolean { + return isNaN(Number(x)); +} + export function wait(ms: number): Promise { return new Promise(resolve => { setTimeout(resolve, ms); @@ -228,29 +234,29 @@ export function parseList(list: string): string[] { // ---------------------------- -export function formatInteger(n: number): string { +export function formatInteger(n: NumberLike): string { return numeral(n).format('0,0'); } -export function formatBytes(n: number): string { +export function formatBytes(n: NumberLike): string { return numeral(n).format('0.00 b'); } -export function formatBytesCompact(n: number): string { +export function formatBytesCompact(n: NumberLike): string { return numeral(n).format('0.00b'); } -export function formatMegabytes(n: number): string { - return numeral(n / 1048576).format('0,0.0'); +export function formatMegabytes(n: NumberLike): string { + return numeral(Number(n) / 1048576).format('0,0.0'); } -export function formatPercent(n: number): string { - return (n * 100).toFixed(2) + '%'; +export function formatPercent(n: NumberLike): string { + return (Number(n) * 100).toFixed(2) + '%'; } -export function formatMillions(n: number): string { - const s = (n / 1e6).toFixed(3); - if (s === '0.000') return String(Math.round(n)); +export function formatMillions(n: NumberLike): string { + const s = (Number(n) / 1e6).toFixed(3); + if (s === '0.000') return String(Math.round(Number(n))); return s + ' M'; } @@ -258,14 +264,15 @@ function pad2(str: string | number): string { return ('00' + str).substr(-2); } -export function formatDuration(ms: number): string { - const timeInHours = Math.floor(ms / 3600000); - const timeInMin = Math.floor(ms / 60000) % 60; - const timeInSec = Math.floor(ms / 1000) % 60; +export function formatDuration(ms: NumberLike): string { + const n = Number(ms); + const timeInHours = Math.floor(n / 3600000); + const timeInMin = Math.floor(n / 60000) % 60; + const timeInSec = Math.floor(n / 1000) % 60; return timeInHours + ':' + pad2(timeInMin) + ':' + pad2(timeInSec); } -export function pluralIfNeeded(n: number, singular: string, plural?: string): string { +export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string): string { if (!plural) plural = singular + 's'; return `${formatInteger(n)} ${n === 1 ? singular : plural}`; } @@ -274,7 +281,7 @@ export function pluralIfNeeded(n: number, singular: string, plural?: string): st export function parseJson(json: string): any { try { - return JSON.parse(json); + return JSONBig.parse(json); } catch (e) { return undefined; } @@ -282,7 +289,7 @@ export function parseJson(json: string): any { export function validJson(json: string): boolean { try { - JSON.parse(json); + JSONBig.parse(json); return true; } catch (e) { return false; diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx index 2901733e164..e9aff6ca1c9 100644 --- a/web-console/src/views/datasource-view/datasource-view.tsx +++ b/web-console/src/views/datasource-view/datasource-view.tsx @@ -57,8 +57,10 @@ import { formatMillions, formatPercent, getDruidErrorMessage, + isNumberLikeNaN, LocalStorageKeys, lookupBy, + NumberLike, pluralIfNeeded, queryDruidSql, QueryManager, @@ -114,7 +116,7 @@ const tableColumns: Record = { const DEFAULT_RULES_KEY = '_default'; -function formatLoadDrop(segmentsToLoad: number, segmentsToDrop: number): string { +function formatLoadDrop(segmentsToLoad: NumberLike, segmentsToDrop: NumberLike): string { const loadDrop: string[] = []; if (segmentsToLoad) { loadDrop.push(`${pluralIfNeeded(segmentsToLoad, 'segment')} to load`); @@ -152,21 +154,21 @@ const PERCENT_BRACES = [formatPercent(1)]; interface DatasourceQueryResultRow { readonly datasource: string; - readonly num_segments: number; - readonly num_segments_to_load: number; - readonly num_segments_to_drop: number; - readonly minute_aligned_segments: number; - readonly hour_aligned_segments: number; - readonly day_aligned_segments: number; - readonly month_aligned_segments: number; - readonly year_aligned_segments: number; - readonly total_data_size: number; - readonly replicated_size: number; - readonly min_segment_rows: number; - readonly avg_segment_rows: number; - readonly max_segment_rows: number; - readonly total_rows: number; - readonly avg_row_size: number; + readonly num_segments: NumberLike; + readonly num_segments_to_load: NumberLike; + readonly num_segments_to_drop: NumberLike; + readonly minute_aligned_segments: NumberLike; + readonly hour_aligned_segments: NumberLike; + readonly day_aligned_segments: NumberLike; + readonly month_aligned_segments: NumberLike; + readonly year_aligned_segments: NumberLike; + readonly total_data_size: NumberLike; + readonly replicated_size: NumberLike; + readonly min_segment_rows: NumberLike; + readonly avg_segment_rows: NumberLike; + readonly max_segment_rows: NumberLike; + readonly total_rows: NumberLike; + readonly avg_row_size: NumberLike; } function makeEmptyDatasourceQueryResultRow(datasource: string): DatasourceQueryResultRow { @@ -224,7 +226,7 @@ interface RetentionDialogOpenOn { interface CompactionDialogOpenOn { readonly datasource: string; - readonly compactionConfig: CompactionConfig; + readonly compactionConfig?: CompactionConfig; } export interface DatasourcesViewProps { @@ -800,9 +802,9 @@ ORDER BY 1`; getDatasourceActions( datasource: string, - unused: boolean, + unused: boolean | undefined, rules: Rule[], - compactionConfig: CompactionConfig, + compactionConfig: CompactionConfig | undefined, ): BasicAction[] { const { goToQuery, goToTask, capabilities } = this.props; @@ -1032,7 +1034,7 @@ ORDER BY 1`; minWidth: 200, accessor: 'num_segments', Cell: ({ value: num_segments, original }) => { - const { datasource, unused, num_segments_to_load } = original; + const { datasource, unused, num_segments_to_load } = original as Datasource; if (unused) { return ( @@ -1086,7 +1088,7 @@ ORDER BY 1`; filterable: false, minWidth: 100, Cell: ({ original }) => { - const { num_segments_to_load, num_segments_to_drop } = original; + const { num_segments_to_load, num_segments_to_drop } = original as Datasource; return formatLoadDrop(num_segments_to_load, num_segments_to_drop); }, }, @@ -1107,8 +1109,13 @@ ORDER BY 1`; filterable: false, width: 220, Cell: ({ value, original }) => { - const { min_segment_rows, max_segment_rows } = original; - if (isNaN(value) || isNaN(min_segment_rows) || isNaN(max_segment_rows)) return '-'; + const { min_segment_rows, max_segment_rows } = original as Datasource; + if ( + isNumberLikeNaN(value) || + isNumberLikeNaN(min_segment_rows) || + isNumberLikeNaN(max_segment_rows) + ) + return '-'; return ( <> { - if (isNaN(value)) return '-'; + if (isNumberLikeNaN(value)) return '-'; return ; }, }, @@ -1183,7 +1190,7 @@ ORDER BY 1`; filterable: false, width: 100, Cell: ({ value }) => { - if (isNaN(value)) return '-'; + if (isNumberLikeNaN(value)) return '-'; return ; }, }, @@ -1194,7 +1201,7 @@ ORDER BY 1`; filterable: false, width: 100, Cell: ({ value }) => { - if (isNaN(value)) return '-'; + if (isNumberLikeNaN(value)) return '-'; return ( ); @@ -1208,7 +1215,7 @@ ORDER BY 1`; filterable: false, width: 150, Cell: ({ original }) => { - const { datasource, compactionConfig, compactionStatus } = original; + const { datasource, compactionConfig, compactionStatus } = original as Datasource; return ( { - const { compactionStatus } = original; + const { compactionStatus } = original as Datasource; if (!compactionStatus || zeroCompactionStatus(compactionStatus)) { return ( @@ -1296,7 +1303,7 @@ ORDER BY 1`; (compactionStatus && compactionStatus.bytesAwaitingCompaction) || 0, filterable: false, Cell: ({ original }) => { - const { compactionStatus } = original; + const { compactionStatus } = original as Datasource; if (!compactionStatus) { return ; @@ -1318,7 +1325,7 @@ ORDER BY 1`; filterable: false, minWidth: 100, Cell: ({ original }) => { - const { datasource, rules } = original; + const { datasource, rules } = original as Datasource; return ( @@ -1348,7 +1355,7 @@ ORDER BY 1`; width: ACTION_COLUMN_WIDTH, filterable: false, Cell: ({ value: datasource, original }) => { - const { unused, rules, compactionConfig } = original; + const { unused, rules, compactionConfig } = original as Datasource; const datasourceActions = this.getDatasourceActions( datasource, unused, diff --git a/web-console/src/views/query-view/query-output/query-output.tsx b/web-console/src/views/query-view/query-output/query-output.tsx index 504aba21b62..78df2159d28 100644 --- a/web-console/src/views/query-view/query-output/query-output.tsx +++ b/web-console/src/views/query-view/query-output/query-output.tsx @@ -33,7 +33,14 @@ import ReactTable from 'react-table'; import { BracedText, TableCell } from '../../../components'; import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog'; -import { copyAndAlert, deepSet, filterMap, prettyPrintSql, stringifyValue } from '../../../utils'; +import { + copyAndAlert, + deepSet, + filterMap, + oneOf, + prettyPrintSql, + stringifyValue, +} from '../../../utils'; import { BasicAction, basicActionsToMenu } from '../../../utils/basic-action'; import { ColumnRenameInput } from './column-rename-input/column-rename-input'; @@ -65,7 +72,7 @@ function getNumericColumnBraces( const numColumns = queryResult.header.length; for (let c = 0; c < numColumns; c++) { const brace = filterMap(rows, row => - typeof row[c] === 'number' ? String(row[c]) : undefined, + oneOf(typeof row[c], 'number', 'bigint') ? String(row[c]) : undefined, ); if (rows.length === brace.length) { numericColumnBraces[c] = brace; diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index 33f03ddc824..fa86426a5cc 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -51,6 +51,7 @@ import { getNeedleAndMode, LocalStorageKeys, makeBooleanFilter, + NumberLike, queryDruidSql, QueryManager, QueryState, @@ -144,7 +145,7 @@ interface SegmentQueryResultRow { partitioning: string; size: number; partition_num: number; - num_rows: number; + num_rows: NumberLike; num_replicas: number; is_available: number; is_published: number; diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap old mode 100755 new mode 100644 index a9e2be893d5..a7238af9b07 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`services view action services view 1`] = ` +exports[`ServicesView renders data 1`] = `
@@ -207,7 +207,40 @@ exports[`services view action services view 1`] = ` }, ] } - data={Array []} + data={ + Array [ + Array [ + Object { + "curr_size": 0, + "host": "localhost", + "is_leader": 0, + "max_size": 0, + "plaintext_port": 8082, + "rank": 5, + "service": "localhost:8082", + "service_type": "broker", + "tier": null, + "tls_port": -1, + }, + Object { + "curr_size": 179744287, + "host": "localhost", + "is_leader": 0, + "max_size": 3000000000n, + "plaintext_port": 8083, + "rank": 4, + "segmentsToDrop": 0, + "segmentsToDropSize": 0, + "segmentsToLoad": 0, + "segmentsToLoadSize": 0, + "service": "localhost:8083", + "service_type": "historical", + "tier": "_default_tier", + "tls_port": -1, + }, + ], + ] + } defaultExpanded={Object {}} defaultFilterMethod={[Function]} defaultFiltered={Array []} @@ -252,7 +285,7 @@ exports[`services view action services view 1`] = ` getTrProps={[Function]} groupedByPivotKey="_groupedByPivot" indexKey="_index" - loading={true} + loading={false} loadingText="Loading..." multiSort={true} nestingLevelKey="_nestingLevel" diff --git a/web-console/src/views/services-view/services-view.spec.tsx b/web-console/src/views/services-view/services-view.spec.tsx index 32a5a6ef412..43367392bfc 100644 --- a/web-console/src/views/services-view/services-view.spec.tsx +++ b/web-console/src/views/services-view/services-view.spec.tsx @@ -19,15 +19,75 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { Capabilities } from '../../utils'; +import { Capabilities, QueryState } from '../../utils'; import { ServicesView } from './services-view'; -describe('services view', () => { - it('action services view', () => { - const servicesView = shallow( - {}} goToTask={() => {}} capabilities={Capabilities.FULL} />, +jest.mock('../../utils', () => { + const originalUtils = jest.requireActual('../../utils'); + + class QueryManagerMock { + private readonly onStateChange: any; + + constructor(opt: { onStateChange: any }) { + this.onStateChange = opt.onStateChange; + } + + public runQuery() { + this.onStateChange( + new QueryState({ + data: [ + [ + { + service: 'localhost:8082', + service_type: 'broker', + tier: null, + host: 'localhost', + plaintext_port: 8082, + tls_port: -1, + curr_size: 0, + max_size: 0, + is_leader: 0, + rank: 5, + }, + { + service: 'localhost:8083', + service_type: 'historical', + tier: '_default_tier', + host: 'localhost', + plaintext_port: 8083, + tls_port: -1, + curr_size: 179744287, + max_size: BigInt(3000000000), + is_leader: 0, + rank: 4, + segmentsToLoad: 0, + segmentsToDrop: 0, + segmentsToLoadSize: 0, + segmentsToDropSize: 0, + }, + ], + ], + }) as any, + ); + } + + public terminate() {} + } + + return { + ...originalUtils, + QueryManager: QueryManagerMock, + }; +}); + +describe('ServicesView', () => { + it('renders data', () => { + const comp = ( + {}} goToTask={() => {}} capabilities={Capabilities.FULL} /> ); + + const servicesView = shallow(comp); expect(servicesView).toMatchSnapshot(); }); }); diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 0f9df1a8e8d..8a90090054a 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -43,7 +43,9 @@ import { formatBytesCompact, LocalStorageKeys, lookupBy, + NumberLike, oneOf, + pluralIfNeeded, queryDruidSql, QueryManager, QueryState, @@ -73,20 +75,24 @@ const tableColumns: Record = { }; function formatQueues( - segmentsToLoad: number, - segmentsToLoadSize: number, - segmentsToDrop: number, - segmentsToDropSize: number, + segmentsToLoad: NumberLike, + segmentsToLoadSize: NumberLike, + segmentsToDrop: NumberLike, + segmentsToDropSize: NumberLike, ): string { const queueParts: string[] = []; if (segmentsToLoad) { queueParts.push( - `${segmentsToLoad} segments to load (${formatBytesCompact(segmentsToLoadSize)})`, + `${pluralIfNeeded(segmentsToLoad, 'segment')} to load (${formatBytesCompact( + segmentsToLoadSize, + )})`, ); } if (segmentsToDrop) { queueParts.push( - `${segmentsToDrop} segments to drop (${formatBytesCompact(segmentsToDropSize)})`, + `${pluralIfNeeded(segmentsToDrop, 'segment')} to drop (${formatBytesCompact( + segmentsToDropSize, + )})`, ); } return queueParts.join(', ') || 'Empty load/drop queues'; @@ -110,38 +116,38 @@ export interface ServicesViewState { } interface ServiceQueryResultRow { - service: string; - service_type: string; - tier: string; - is_leader: number; - curr_size: number; - host: string; - max_size: number; - plaintext_port: number; - tls_port: number; + readonly service: string; + readonly service_type: string; + readonly tier: string; + readonly is_leader: number; + readonly host: string; + readonly curr_size: NumberLike; + readonly max_size: NumberLike; + readonly plaintext_port: number; + readonly tls_port: number; } interface LoadQueueStatus { - segmentsToDrop: number; - segmentsToDropSize: number; - segmentsToLoad: number; - segmentsToLoadSize: number; + readonly segmentsToDrop: NumberLike; + readonly segmentsToDropSize: NumberLike; + readonly segmentsToLoad: NumberLike; + readonly segmentsToLoadSize: NumberLike; } interface MiddleManagerQueryResultRow { - availabilityGroups: string[]; - blacklistedUntil: string | null; - currCapacityUsed: number; - lastCompletedTaskTime: string; - category: string; - runningTasks: string[]; - worker: { - capacity: number; - host: string; - ip: string; - scheme: string; - version: string; - category: string; + readonly availabilityGroups: string[]; + readonly blacklistedUntil: string | null; + readonly currCapacityUsed: NumberLike; + readonly lastCompletedTaskTime: string; + readonly category: string; + readonly runningTasks: string[]; + readonly worker: { + readonly capacity: NumberLike; + readonly host: string; + readonly ip: string; + readonly scheme: string; + readonly version: string; + readonly category: string; }; } @@ -164,7 +170,15 @@ export class ServicesView extends React.PureComponent 1 static SERVICE_SQL = `SELECT - "server" AS "service", "server_type" AS "service_type", "tier", "host", "plaintext_port", "tls_port", "curr_size", "max_size", "is_leader", + "server" AS "service", + "server_type" AS "service_type", + "tier", + "host", + "plaintext_port", + "tls_port", + "curr_size", + "max_size", + "is_leader", ( CASE "server_type" WHEN 'coordinator' THEN 8 @@ -430,26 +444,30 @@ ORDER BY "rank" DESC, "service" DESC`; filterable: false, accessor: row => { if (oneOf(row.service_type, 'middle_manager', 'indexer')) { - return row.worker ? (row.currCapacityUsed || 0) / row.worker.capacity : null; + return row.worker + ? (Number(row.currCapacityUsed) || 0) / Number(row.worker.capacity) + : null; } else { - return row.max_size ? row.curr_size / row.max_size : null; + return row.max_size ? Number(row.curr_size) / Number(row.max_size) : null; } }, Aggregated: row => { switch (row.row._pivotVal) { case 'historical': { - const originalHistoricals = row.subRows.map(r => r._original); - const totalCurr = sum(originalHistoricals, s => s.curr_size); - const totalMax = sum(originalHistoricals, s => s.max_size); + const originalHistoricals: ServiceResultRow[] = row.subRows.map(r => r._original); + const totalCurr = sum(originalHistoricals, s => Number(s.curr_size)); + const totalMax = sum(originalHistoricals, s => Number(s.max_size)); return fillIndicator(totalCurr / totalMax); } case 'indexer': case 'middle_manager': { - const originalMiddleManagers = row.subRows.map(r => r._original); + const originalMiddleManagers: ServiceResultRow[] = row.subRows.map( + r => r._original, + ); const totalCurrCapacityUsed = sum( originalMiddleManagers, - s => s.currCapacityUsed || 0, + s => Number(s.currCapacityUsed) || 0, ); const totalWorkerCapacity = sum( originalMiddleManagers, @@ -506,7 +524,7 @@ ORDER BY "rank" DESC, "service" DESC`; } else if (oneOf(row.service_type, 'coordinator', 'overlord')) { return (row.is_leader || 0) === 1 ? 'leader' : ''; } else { - return (row.segmentsToLoad || 0) + (row.segmentsToDrop || 0); + return (Number(row.segmentsToLoad) || 0) + (Number(row.segmentsToDrop) || 0); } }, Cell: row => { @@ -542,11 +560,11 @@ ORDER BY "rank" DESC, "service" DESC`; }, Aggregated: row => { if (row.row._pivotVal !== 'historical') return ''; - const originals = row.subRows.map(r => r._original); - const segmentsToLoad = sum(originals, s => s.segmentsToLoad); - const segmentsToLoadSize = sum(originals, s => s.segmentsToLoadSize); - const segmentsToDrop = sum(originals, s => s.segmentsToDrop); - const segmentsToDropSize = sum(originals, s => s.segmentsToDropSize); + const originals: ServiceResultRow[] = row.subRows.map(r => r._original); + const segmentsToLoad = sum(originals, s => Number(s.segmentsToLoad) || 0); + const segmentsToLoadSize = sum(originals, s => Number(s.segmentsToLoadSize) || 0); + const segmentsToDrop = sum(originals, s => Number(s.segmentsToDrop) || 0); + const segmentsToDropSize = sum(originals, s => Number(s.segmentsToDropSize) || 0); return formatQueues( segmentsToLoad, segmentsToLoadSize,