Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: better handle BigInt math (#11450) 

* better handle BigInt math

* correctly brace bigint

* feedback fixes and tests
This commit is contained in:
Vadim Ogievetsky 2021-07-20 17:17:19 -07:00 committed by GitHub
parent 1937b5c0da
commit aee2f2e24f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 245 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
web-console/src/utils/general.tsx
View File

@ -33,6 +33,12 @@ import { AppToaster } from '../singletons';
export const EMPTY_OBJECT: any = {};
export const EMPTY_ARRAY: any[] = [];
export type NumberLike = number | BigInt;
export function isNumberLikeNaN(x: NumberLike): boolean {
return isNaN(Number(x));
}
export function wait(ms: number): Promise<void> {
return new Promise(resolve => {
setTimeout(resolve, ms);
@ -228,29 +234,29 @@ export function parseList(list: string): string[] {
// ----------------------------
export function formatInteger(n: number): string {
export function formatInteger(n: NumberLike): string {
return numeral(n).format('0,0');
}
export function formatBytes(n: number): string {
export function formatBytes(n: NumberLike): string {
return numeral(n).format('0.00 b');
}
export function formatBytesCompact(n: number): string {
export function formatBytesCompact(n: NumberLike): string {
return numeral(n).format('0.00b');
}
export function formatMegabytes(n: number): string {
return numeral(n / 1048576).format('0,0.0');
export function formatMegabytes(n: NumberLike): string {
return numeral(Number(n) / 1048576).format('0,0.0');
}
export function formatPercent(n: number): string {
return (n * 100).toFixed(2) + '%';
export function formatPercent(n: NumberLike): string {
return (Number(n) * 100).toFixed(2) + '%';
}
export function formatMillions(n: number): string {
const s = (n / 1e6).toFixed(3);
if (s === '0.000') return String(Math.round(n));
export function formatMillions(n: NumberLike): string {
const s = (Number(n) / 1e6).toFixed(3);
if (s === '0.000') return String(Math.round(Number(n)));
return s + ' M';
}
@ -258,14 +264,15 @@ function pad2(str: string | number): string {
return ('00' + str).substr(-2);
}
export function formatDuration(ms: number): string {
const timeInHours = Math.floor(ms / 3600000);
const timeInMin = Math.floor(ms / 60000) % 60;
const timeInSec = Math.floor(ms / 1000) % 60;
export function formatDuration(ms: NumberLike): string {
const n = Number(ms);
const timeInHours = Math.floor(n / 3600000);
const timeInMin = Math.floor(n / 60000) % 60;
const timeInSec = Math.floor(n / 1000) % 60;
return timeInHours + ':' + pad2(timeInMin) + ':' + pad2(timeInSec);
}
export function pluralIfNeeded(n: number, singular: string, plural?: string): string {
export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string): string {
if (!plural) plural = singular + 's';
return `${formatInteger(n)} ${n === 1 ? singular : plural}`;
}
@ -274,7 +281,7 @@ export function pluralIfNeeded(n: number, singular: string, plural?: string): st
export function parseJson(json: string): any {
try {
return JSON.parse(json);
return JSONBig.parse(json);
} catch (e) {
return undefined;
}
@ -282,7 +289,7 @@ export function parseJson(json: string): any {
export function validJson(json: string): boolean {
try {
JSON.parse(json);
JSONBig.parse(json);
return true;
} catch (e) {
return false;

83
web-console/src/views/datasource-view/datasource-view.tsx
View File

@ -57,8 +57,10 @@ import {
formatMillions,
formatPercent,
getDruidErrorMessage,
isNumberLikeNaN,
LocalStorageKeys,
lookupBy,
NumberLike,
pluralIfNeeded,
queryDruidSql,
QueryManager,
@ -114,7 +116,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
const DEFAULT_RULES_KEY = '_default';
function formatLoadDrop(segmentsToLoad: number, segmentsToDrop: number): string {
function formatLoadDrop(segmentsToLoad: NumberLike, segmentsToDrop: NumberLike): string {
const loadDrop: string[] = [];
if (segmentsToLoad) {
loadDrop.push(`${pluralIfNeeded(segmentsToLoad, 'segment')} to load`);
@ -152,21 +154,21 @@ const PERCENT_BRACES = [formatPercent(1)];
interface DatasourceQueryResultRow {
readonly datasource: string;
readonly num_segments: number;
readonly num_segments_to_load: number;
readonly num_segments_to_drop: number;
readonly minute_aligned_segments: number;
readonly hour_aligned_segments: number;
readonly day_aligned_segments: number;
readonly month_aligned_segments: number;
readonly year_aligned_segments: number;
readonly total_data_size: number;
readonly replicated_size: number;
readonly min_segment_rows: number;
readonly avg_segment_rows: number;
readonly max_segment_rows: number;
readonly total_rows: number;
readonly avg_row_size: number;
readonly num_segments: NumberLike;
readonly num_segments_to_load: NumberLike;
readonly num_segments_to_drop: NumberLike;
readonly minute_aligned_segments: NumberLike;
readonly hour_aligned_segments: NumberLike;
readonly day_aligned_segments: NumberLike;
readonly month_aligned_segments: NumberLike;
readonly year_aligned_segments: NumberLike;
readonly total_data_size: NumberLike;
readonly replicated_size: NumberLike;
readonly min_segment_rows: NumberLike;
readonly avg_segment_rows: NumberLike;
readonly max_segment_rows: NumberLike;
readonly total_rows: NumberLike;
readonly avg_row_size: NumberLike;
}
function makeEmptyDatasourceQueryResultRow(datasource: string): DatasourceQueryResultRow {
@ -224,7 +226,7 @@ interface RetentionDialogOpenOn {
interface CompactionDialogOpenOn {
readonly datasource: string;
readonly compactionConfig: CompactionConfig;
readonly compactionConfig?: CompactionConfig;
}
export interface DatasourcesViewProps {
@ -800,9 +802,9 @@ ORDER BY 1`;
getDatasourceActions(
datasource: string,
unused: boolean,
unused: boolean | undefined,
rules: Rule[],
compactionConfig: CompactionConfig,
compactionConfig: CompactionConfig | undefined,
): BasicAction[] {
const { goToQuery, goToTask, capabilities } = this.props;
@ -1032,7 +1034,7 @@ ORDER BY 1`;
minWidth: 200,
accessor: 'num_segments',
Cell: ({ value: num_segments, original }) => {
const { datasource, unused, num_segments_to_load } = original;
const { datasource, unused, num_segments_to_load } = original as Datasource;
if (unused) {
return (
<span>
@ -1086,7 +1088,7 @@ ORDER BY 1`;
filterable: false,
minWidth: 100,
Cell: ({ original }) => {
const { num_segments_to_load, num_segments_to_drop } = original;
const { num_segments_to_load, num_segments_to_drop } = original as Datasource;
return formatLoadDrop(num_segments_to_load, num_segments_to_drop);
},
},
@ -1107,8 +1109,13 @@ ORDER BY 1`;
filterable: false,
width: 220,
Cell: ({ value, original }) => {
const { min_segment_rows, max_segment_rows } = original;
if (isNaN(value) || isNaN(min_segment_rows) || isNaN(max_segment_rows)) return '-';
const { min_segment_rows, max_segment_rows } = original as Datasource;
if (
isNumberLikeNaN(value) ||
isNumberLikeNaN(min_segment_rows) ||
isNumberLikeNaN(max_segment_rows)
)
return '-';
return (
<>
<BracedText
@ -1141,22 +1148,22 @@ ORDER BY 1`;
day_aligned_segments,
month_aligned_segments,
year_aligned_segments,
} = original;
} = original as Datasource;
const segmentGranularities: string[] = [];
if (!num_segments || isNaN(year_aligned_segments)) return '-';
if (num_segments - minute_aligned_segments) {
if (!num_segments || isNumberLikeNaN(year_aligned_segments)) return '-';
if (num_segments !== minute_aligned_segments) {
segmentGranularities.push('Sub minute');
}
if (minute_aligned_segments - hour_aligned_segments) {
if (minute_aligned_segments !== hour_aligned_segments) {
segmentGranularities.push('Minute');
}
if (hour_aligned_segments - day_aligned_segments) {
if (hour_aligned_segments !== day_aligned_segments) {
segmentGranularities.push('Hour');
}
if (day_aligned_segments - month_aligned_segments) {
if (day_aligned_segments !== month_aligned_segments) {
segmentGranularities.push('Day');
}
if (month_aligned_segments - year_aligned_segments) {
if (month_aligned_segments !== year_aligned_segments) {
segmentGranularities.push('Month');
}
if (year_aligned_segments) {
@ -1172,7 +1179,7 @@ ORDER BY 1`;
filterable: false,
width: 100,
Cell: ({ value }) => {
if (isNaN(value)) return '-';
if (isNumberLikeNaN(value)) return '-';
return <BracedText text={formatTotalRows(value)} braces={totalRowsValues} />;
},
},
@ -1183,7 +1190,7 @@ ORDER BY 1`;
filterable: false,
width: 100,
Cell: ({ value }) => {
if (isNaN(value)) return '-';
if (isNumberLikeNaN(value)) return '-';
return <BracedText text={formatAvgRowSize(value)} braces={avgRowSizeValues} />;
},
},
@ -1194,7 +1201,7 @@ ORDER BY 1`;
filterable: false,
width: 100,
Cell: ({ value }) => {
if (isNaN(value)) return '-';
if (isNumberLikeNaN(value)) return '-';
return (
<BracedText text={formatReplicatedSize(value)} braces={replicatedSizeValues} />
);
@ -1208,7 +1215,7 @@ ORDER BY 1`;
filterable: false,
width: 150,
Cell: ({ original }) => {
const { datasource, compactionConfig, compactionStatus } = original;
const { datasource, compactionConfig, compactionStatus } = original as Datasource;
return (
<span
className="clickable-cell"
@ -1239,7 +1246,7 @@ ORDER BY 1`;
: 0,
filterable: false,
Cell: ({ original }) => {
const { compactionStatus } = original;
const { compactionStatus } = original as Datasource;
if (!compactionStatus || zeroCompactionStatus(compactionStatus)) {
return (
@ -1296,7 +1303,7 @@ ORDER BY 1`;
(compactionStatus && compactionStatus.bytesAwaitingCompaction) || 0,
filterable: false,
Cell: ({ original }) => {
const { compactionStatus } = original;
const { compactionStatus } = original as Datasource;
if (!compactionStatus) {
return <BracedText text="-" braces={leftToBeCompactedValues} />;
@ -1318,7 +1325,7 @@ ORDER BY 1`;
filterable: false,
minWidth: 100,
Cell: ({ original }) => {
const { datasource, rules } = original;
const { datasource, rules } = original as Datasource;
return (
<span
onClick={() =>
@ -1348,7 +1355,7 @@ ORDER BY 1`;
width: ACTION_COLUMN_WIDTH,
filterable: false,
Cell: ({ value: datasource, original }) => {
const { unused, rules, compactionConfig } = original;
const { unused, rules, compactionConfig } = original as Datasource;
const datasourceActions = this.getDatasourceActions(
datasource,
unused,

11
web-console/src/views/query-view/query-output/query-output.tsx
View File

@ -33,7 +33,14 @@ import ReactTable from 'react-table';
import { BracedText, TableCell } from '../../../components';
import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog';
import { copyAndAlert, deepSet, filterMap, prettyPrintSql, stringifyValue } from '../../../utils';
import {
copyAndAlert,
deepSet,
filterMap,
oneOf,
prettyPrintSql,
stringifyValue,
} from '../../../utils';
import { BasicAction, basicActionsToMenu } from '../../../utils/basic-action';
import { ColumnRenameInput } from './column-rename-input/column-rename-input';
@ -65,7 +72,7 @@ function getNumericColumnBraces(
const numColumns = queryResult.header.length;
for (let c = 0; c < numColumns; c++) {
const brace = filterMap(rows, row =>
typeof row[c] === 'number' ? String(row[c]) : undefined,
oneOf(typeof row[c], 'number', 'bigint') ? String(row[c]) : undefined,
);
if (rows.length === brace.length) {
numericColumnBraces[c] = brace;

3
web-console/src/views/segments-view/segments-view.tsx
View File

@ -51,6 +51,7 @@ import {
getNeedleAndMode,
LocalStorageKeys,
makeBooleanFilter,
NumberLike,
queryDruidSql,
QueryManager,
QueryState,
@ -144,7 +145,7 @@ interface SegmentQueryResultRow {
partitioning: string;
size: number;
partition_num: number;
num_rows: number;
num_rows: NumberLike;
num_replicas: number;
is_available: number;
is_published: number;

39
web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap Executable file → Normal file
View File

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`services view action services view 1`] = `
exports[`ServicesView renders data 1`] = `
<div
className="services-view app-view"
>
@ -207,7 +207,40 @@ exports[`services view action services view 1`] = `
},
]
}
data={Array []}
data={
Array [
Array [
Object {
"curr_size": 0,
"host": "localhost",
"is_leader": 0,
"max_size": 0,
"plaintext_port": 8082,
"rank": 5,
"service": "localhost:8082",
"service_type": "broker",
"tier": null,
"tls_port": -1,
},
Object {
"curr_size": 179744287,
"host": "localhost",
"is_leader": 0,
"max_size": 3000000000n,
"plaintext_port": 8083,
"rank": 4,
"segmentsToDrop": 0,
"segmentsToDropSize": 0,
"segmentsToLoad": 0,
"segmentsToLoadSize": 0,
"service": "localhost:8083",
"service_type": "historical",
"tier": "_default_tier",
"tls_port": -1,
},
],
]
}
defaultExpanded={Object {}}
defaultFilterMethod={[Function]}
defaultFiltered={Array []}
@ -252,7 +285,7 @@ exports[`services view action services view 1`] = `
getTrProps={[Function]}
groupedByPivotKey="_groupedByPivot"
indexKey="_index"
loading={true}
loading={false}
loadingText="Loading..."
multiSort={true}
nestingLevelKey="_nestingLevel"

70
web-console/src/views/services-view/services-view.spec.tsx
View File

@ -19,15 +19,75 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities, QueryState } from '../../utils';
import { ServicesView } from './services-view';
describe('services view', () => {
it('action services view', () => {
const servicesView = shallow(
<ServicesView goToQuery={() => {}} goToTask={() => {}} capabilities={Capabilities.FULL} />,
jest.mock('../../utils', () => {
const originalUtils = jest.requireActual('../../utils');
class QueryManagerMock {
private readonly onStateChange: any;
constructor(opt: { onStateChange: any }) {
this.onStateChange = opt.onStateChange;
}
public runQuery() {
this.onStateChange(
new QueryState({
data: [
[
{
service: 'localhost:8082',
service_type: 'broker',
tier: null,
host: 'localhost',
plaintext_port: 8082,
tls_port: -1,
curr_size: 0,
max_size: 0,
is_leader: 0,
rank: 5,
},
{
service: 'localhost:8083',
service_type: 'historical',
tier: '_default_tier',
host: 'localhost',
plaintext_port: 8083,
tls_port: -1,
curr_size: 179744287,
max_size: BigInt(3000000000),
is_leader: 0,
rank: 4,
segmentsToLoad: 0,
segmentsToDrop: 0,
segmentsToLoadSize: 0,
segmentsToDropSize: 0,
},
],
],
}) as any,
);
}
public terminate() {}
}
return {
...originalUtils,
QueryManager: QueryManagerMock,
};
});
describe('ServicesView', () => {
it('renders data', () => {
const comp = (
<ServicesView goToQuery={() => {}} goToTask={() => {}} capabilities={Capabilities.FULL} />
);
const servicesView = shallow(comp);
expect(servicesView).toMatchSnapshot();
});
});

110
web-console/src/views/services-view/services-view.tsx
View File

@ -43,7 +43,9 @@ import {
formatBytesCompact,
LocalStorageKeys,
lookupBy,
NumberLike,
oneOf,
pluralIfNeeded,
queryDruidSql,
QueryManager,
QueryState,
@ -73,20 +75,24 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
};
function formatQueues(
segmentsToLoad: number,
segmentsToLoadSize: number,
segmentsToDrop: number,
segmentsToDropSize: number,
segmentsToLoad: NumberLike,
segmentsToLoadSize: NumberLike,
segmentsToDrop: NumberLike,
segmentsToDropSize: NumberLike,
): string {
const queueParts: string[] = [];
if (segmentsToLoad) {
queueParts.push(
`${segmentsToLoad} segments to load (${formatBytesCompact(segmentsToLoadSize)})`,
`${pluralIfNeeded(segmentsToLoad, 'segment')} to load (${formatBytesCompact(
segmentsToLoadSize,
)})`,
);
}
if (segmentsToDrop) {
queueParts.push(
`${segmentsToDrop} segments to drop (${formatBytesCompact(segmentsToDropSize)})`,
`${pluralIfNeeded(segmentsToDrop, 'segment')} to drop (${formatBytesCompact(
segmentsToDropSize,
)})`,
);
}
return queueParts.join(', ') || 'Empty load/drop queues';
@ -110,38 +116,38 @@ export interface ServicesViewState {
}
interface ServiceQueryResultRow {
service: string;
service_type: string;
tier: string;
is_leader: number;
curr_size: number;
host: string;
max_size: number;
plaintext_port: number;
tls_port: number;
readonly service: string;
readonly service_type: string;
readonly tier: string;
readonly is_leader: number;
readonly host: string;
readonly curr_size: NumberLike;
readonly max_size: NumberLike;
readonly plaintext_port: number;
readonly tls_port: number;
}
interface LoadQueueStatus {
segmentsToDrop: number;
segmentsToDropSize: number;
segmentsToLoad: number;
segmentsToLoadSize: number;
readonly segmentsToDrop: NumberLike;
readonly segmentsToDropSize: NumberLike;
readonly segmentsToLoad: NumberLike;
readonly segmentsToLoadSize: NumberLike;
}
interface MiddleManagerQueryResultRow {
availabilityGroups: string[];
blacklistedUntil: string | null;
currCapacityUsed: number;
lastCompletedTaskTime: string;
category: string;
runningTasks: string[];
worker: {
capacity: number;
host: string;
ip: string;
scheme: string;
version: string;
category: string;
readonly availabilityGroups: string[];
readonly blacklistedUntil: string | null;
readonly currCapacityUsed: NumberLike;
readonly lastCompletedTaskTime: string;
readonly category: string;
readonly runningTasks: string[];
readonly worker: {
readonly capacity: NumberLike;
readonly host: string;
readonly ip: string;
readonly scheme: string;
readonly version: string;
readonly category: string;
};
}
@ -164,7 +170,15 @@ export class ServicesView extends React.PureComponent<ServicesViewProps, Service
// peon => 1
static SERVICE_SQL = `SELECT
"server" AS "service", "server_type" AS "service_type", "tier", "host", "plaintext_port", "tls_port", "curr_size", "max_size", "is_leader",
"server" AS "service",
"server_type" AS "service_type",
"tier",
"host",
"plaintext_port",
"tls_port",
"curr_size",
"max_size",
"is_leader",
(
CASE "server_type"
WHEN 'coordinator' THEN 8
@ -430,26 +444,30 @@ ORDER BY "rank" DESC, "service" DESC`;
filterable: false,
accessor: row => {
if (oneOf(row.service_type, 'middle_manager', 'indexer')) {
return row.worker ? (row.currCapacityUsed || 0) / row.worker.capacity : null;
return row.worker
? (Number(row.currCapacityUsed) || 0) / Number(row.worker.capacity)
: null;
} else {
return row.max_size ? row.curr_size / row.max_size : null;
return row.max_size ? Number(row.curr_size) / Number(row.max_size) : null;
}
},
Aggregated: row => {
switch (row.row._pivotVal) {
case 'historical': {
const originalHistoricals = row.subRows.map(r => r._original);
const totalCurr = sum(originalHistoricals, s => s.curr_size);
const totalMax = sum(originalHistoricals, s => s.max_size);
const originalHistoricals: ServiceResultRow[] = row.subRows.map(r => r._original);
const totalCurr = sum(originalHistoricals, s => Number(s.curr_size));
const totalMax = sum(originalHistoricals, s => Number(s.max_size));
return fillIndicator(totalCurr / totalMax);
}
case 'indexer':
case 'middle_manager': {
const originalMiddleManagers = row.subRows.map(r => r._original);
const originalMiddleManagers: ServiceResultRow[] = row.subRows.map(
r => r._original,
);
const totalCurrCapacityUsed = sum(
originalMiddleManagers,
s => s.currCapacityUsed || 0,
s => Number(s.currCapacityUsed) || 0,
);
const totalWorkerCapacity = sum(
originalMiddleManagers,
@ -506,7 +524,7 @@ ORDER BY "rank" DESC, "service" DESC`;
} else if (oneOf(row.service_type, 'coordinator', 'overlord')) {
return (row.is_leader || 0) === 1 ? 'leader' : '';
} else {
return (row.segmentsToLoad || 0) + (row.segmentsToDrop || 0);
return (Number(row.segmentsToLoad) || 0) + (Number(row.segmentsToDrop) || 0);
}
},
Cell: row => {
@ -542,11 +560,11 @@ ORDER BY "rank" DESC, "service" DESC`;
},
Aggregated: row => {
if (row.row._pivotVal !== 'historical') return '';
const originals = row.subRows.map(r => r._original);
const segmentsToLoad = sum(originals, s => s.segmentsToLoad);
const segmentsToLoadSize = sum(originals, s => s.segmentsToLoadSize);
const segmentsToDrop = sum(originals, s => s.segmentsToDrop);
const segmentsToDropSize = sum(originals, s => s.segmentsToDropSize);
const originals: ServiceResultRow[] = row.subRows.map(r => r._original);
const segmentsToLoad = sum(originals, s => Number(s.segmentsToLoad) || 0);
const segmentsToLoadSize = sum(originals, s => Number(s.segmentsToLoadSize) || 0);
const segmentsToDrop = sum(originals, s => Number(s.segmentsToDrop) || 0);
const segmentsToDropSize = sum(originals, s => Number(s.segmentsToDropSize) || 0);
return formatQueues(
segmentsToLoad,
segmentsToLoadSize,