Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make web console fast around sys.segments (#10909) 

* do not load all the segments

* fix filtering

* update datasource view

* updated tests

* remove trimmedSegments

* Availability detail

* be smart about when showing smart modes

* fix tests

* add coordinator overlord mode
This commit is contained in:
Vadim Ogievetsky 2021-03-10 19:59:50 -08:00 committed by GitHub
parent 43638cc6f9
commit 4897731e37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 567 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
View File

@ -119,6 +119,42 @@ exports[`header bar matches snapshot 1`] = `
shouldDismissPopover={true}
text="Lookups"
/>
<Blueprint3.MenuDivider />
<Blueprint3.MenuItem
disabled={false}
icon="cog"
multiline={false}
popoverProps={Object {}}
shouldDismissPopover={true}
text="Console options"
>
<React.Fragment>
<Blueprint3.MenuItem
disabled={false}
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
shouldDismissPopover={true}
text="Force Coordinator/Overlord mode"
/>
<Blueprint3.MenuItem
disabled={false}
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
shouldDismissPopover={true}
text="Force Coordinator mode"
/>
<Blueprint3.MenuItem
disabled={false}
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
shouldDismissPopover={true}
text="Force Overlord mode"
/>
</React.Fragment>
</Blueprint3.MenuItem>
</Blueprint3.Menu>
}
defaultIsOpen={false}

59
web-console/src/components/header-bar/header-bar.tsx
View File

@ -22,6 +22,7 @@ import {
Button,
Intent,
Menu,
MenuDivider,
MenuItem,
Navbar,
NavbarDivider,
@ -39,12 +40,20 @@ import {
OverlordDynamicConfigDialog,
} from '../../dialogs';
import { getLink } from '../../links';
import { Capabilities } from '../../utils';
import {
Capabilities,
localStorageGetJson,
LocalStorageKeys,
localStorageRemove,
localStorageSetJson,
} from '../../utils';
import { ExternalLink } from '../external-link/external-link';
import { PopoverText } from '../popover-text/popover-text';
import './header-bar.scss';
const capabilitiesOverride = localStorageGetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE);
export type HeaderActiveTab =
| null
| 'load-data'
@ -121,6 +130,17 @@ const RestrictedMode = React.memo(function RestrictedMode(props: RestrictedModeP
);
break;
case 'coordinator-overlord':
label = 'Coordinator/Overlord mode';
message = (
<p>
It appears that you are accessing the console on the Coordinator/Overlord shared service.
Due to the lack of access to some APIs on this service the console will operate in a
limited mode. The full version of the console can be accessed on the Router service.
</p>
);
break;
case 'coordinator':
label = 'Coordinator mode';
message = (
@ -216,6 +236,16 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
</Menu>
);
function setForcedMode(capabilities: Capabilities | undefined): void {
if (capabilities) {
localStorageSetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE, capabilities);
} else {
localStorageRemove(LocalStorageKeys.CAPABILITIES_OVERRIDE);
}
location.reload();
}
const capabilitiesMode = capabilities.getModeExtended();
const configMenu = (
<Menu>
<MenuItem
@ -243,6 +273,33 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
href="#lookups"
disabled={!capabilities.hasCoordinatorAccess()}
/>
<MenuDivider />
<MenuItem icon={IconNames.COG} text="Console options">
{capabilitiesOverride ? (
<MenuItem text="Clear forced mode" onClick={() => setForcedMode(undefined)} />
) : (
<>
{capabilitiesMode !== 'coordinator-overlord' && (
<MenuItem
text="Force Coordinator/Overlord mode"
onClick={() => setForcedMode(Capabilities.COORDINATOR_OVERLORD)}
/>
)}
{capabilitiesMode !== 'coordinator' && (
<MenuItem
text="Force Coordinator mode"
onClick={() => setForcedMode(Capabilities.COORDINATOR)}
/>
)}
{capabilitiesMode !== 'overlord' && (
<MenuItem
text="Force Overlord mode"
onClick={() => setForcedMode(Capabilities.OVERLORD)}
/>
)}
</>
)}
</MenuItem>
</Menu>
);

18
web-console/src/components/table-column-selector/table-column-selector.tsx
View File

@ -18,7 +18,7 @@
import { Button, Menu, Popover, Position } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import React, { useState } from 'react';
import { MenuCheckbox } from '../menu-checkbox/menu-checkbox';
@ -27,13 +27,15 @@ import './table-column-selector.scss';
interface TableColumnSelectorProps {
columns: string[];
onChange: (column: string) => void;
onClose?: (added: number) => void;
tableColumnsHidden: string[];
}
export const TableColumnSelector = React.memo(function TableColumnSelector(
props: TableColumnSelectorProps,
) {
const { columns, onChange, tableColumnsHidden } = props;
const { columns, onChange, onClose, tableColumnsHidden } = props;
const [added, setAdded] = useState(0);
const isColumnShown = (column: string) => !tableColumnsHidden.includes(column);
@ -44,7 +46,12 @@ export const TableColumnSelector = React.memo(function TableColumnSelector(
text={column}
key={column}
checked={isColumnShown(column)}
onChange={() => onChange(column)}
onChange={() => {
if (!isColumnShown(column)) {
setAdded(added + 1);
}
onChange(column);
}}
/>
))}
</Menu>
@ -57,6 +64,11 @@ export const TableColumnSelector = React.memo(function TableColumnSelector(
className="table-column-selector"
content={checkboxes}
position={Position.BOTTOM_RIGHT}
onOpened={() => setAdded(0)}
onClose={() => {
if (!onClose) return;
onClose(added);
}}
>
<Button rightIcon={IconNames.CARET_DOWN}>
Columns <span className="counter">{counterText}</span>

2
web-console/src/singletons/api.ts
View File

@ -47,7 +47,7 @@ export class Api {
static encodePath(path: string): string {
return path.replace(
/[?#%&'\[\]]/g,
/[?#%&'\[\]\\]/g,
c =>
'%' +
c

10
web-console/src/utils/capabilities.ts
View File

@ -27,6 +27,7 @@ export type CapabilitiesModeExtended =
| 'no-sql'
| 'no-proxy'
| 'no-sql-no-proxy'
| 'coordinator-overlord'
| 'coordinator'
| 'overlord';
@ -41,6 +42,7 @@ export interface CapabilitiesOptions {
export class Capabilities {
static STATUS_TIMEOUT = 2000;
static FULL: Capabilities;
static COORDINATOR_OVERLORD: Capabilities;
static COORDINATOR: Capabilities;
static OVERLORD: Capabilities;
@ -154,6 +156,9 @@ export class Capabilities {
return 'no-sql-no-proxy';
}
} else {
if (coordinator && overlord) {
return 'coordinator-overlord';
}
if (coordinator) {
return 'coordinator';
}
@ -198,6 +203,11 @@ Capabilities.FULL = new Capabilities({
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR_OVERLORD = new Capabilities({
queryType: 'none',
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR = new Capabilities({
queryType: 'none',
coordinator: true,

9
web-console/src/utils/general.tsx
View File

@ -98,7 +98,8 @@ interface NeedleAndMode {
mode: 'exact' | 'includes';
}
function getNeedleAndMode(input: string): NeedleAndMode {
export function getNeedleAndMode(filter: Filter): NeedleAndMode {
const input = filter.value.toLowerCase();
if (input.startsWith(`"`) && input.endsWith(`"`)) {
return {
needle: input.slice(1, -1),
@ -114,7 +115,7 @@ function getNeedleAndMode(input: string): NeedleAndMode {
export function booleanCustomTableFilter(filter: Filter, value: any): boolean {
if (value == null) return false;
const haystack = String(value).toLowerCase();
const needleAndMode: NeedleAndMode = getNeedleAndMode(filter.value.toLowerCase());
const needleAndMode: NeedleAndMode = getNeedleAndMode(filter);
const needle = needleAndMode.needle;
if (needleAndMode.mode === 'exact') {
return needle === haystack;
@ -123,13 +124,13 @@ export function booleanCustomTableFilter(filter: Filter, value: any): boolean {
}
export function sqlQueryCustomTableFilter(filter: Filter): SqlExpression {
const needleAndMode: NeedleAndMode = getNeedleAndMode(filter.value);
const needleAndMode: NeedleAndMode = getNeedleAndMode(filter);
const needle = needleAndMode.needle;
if (needleAndMode.mode === 'exact') {
return SqlRef.columnWithQuotes(filter.id).equal(SqlLiteral.create(needle));
} else {
return SqlFunction.simple('LOWER', [SqlRef.columnWithQuotes(filter.id)]).like(
SqlLiteral.create(`%${needle.toLowerCase()}%`),
SqlLiteral.create(`%${needle}%`),
);
}
}

5
web-console/src/utils/local-storage-keys.tsx
View File

@ -67,3 +67,8 @@ export function localStorageGetJson(key: LocalStorageKeys): any {
return;
}
}
export function localStorageRemove(key: LocalStorageKeys): void {
if (typeof localStorage === 'undefined') return;
return localStorage.removeItem(key);
}

13
web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
View File

@ -26,7 +26,7 @@ exports[`data source view matches snapshot 1`] = `
}
>
<Blueprint3.MenuItem
disabled={false}
disabled={true}
icon="application"
multiline={false}
onClick={[Function]}
@ -61,7 +61,7 @@ exports[`data source view matches snapshot 1`] = `
Array [
"Datasource name",
"Availability",
"Segment load/drop queues",
"Availability detail",
"Total data size",
"Segment size",
"Segment granularity",
@ -76,6 +76,7 @@ exports[`data source view matches snapshot 1`] = `
]
}
onChange={[Function]}
onClose={[Function]}
tableColumnsHidden={Array []}
/>
</Memo(ViewControlBar)>
@ -150,9 +151,8 @@ exports[`data source view matches snapshot 1`] = `
Object {
"Cell": [Function],
"Header": "Availability",
"accessor": [Function],
"accessor": "num_segments",
"filterable": false,
"id": "availability",
"minWidth": 200,
"show": true,
"sortMethod": [Function],
@ -160,13 +160,12 @@ exports[`data source view matches snapshot 1`] = `
Object {
"Cell": [Function],
"Header": <React.Fragment>
Segment load/drop
Availability
<br />
queues
detail
</React.Fragment>,
"accessor": "num_segments_to_load",
"filterable": false,
"id": "load-drop",
"minWidth": 100,
"show": true,
},

256
web-console/src/views/datasource-view/datasource-view.tsx
View File

@ -49,6 +49,7 @@ import {
addFilter,
Capabilities,
CapabilitiesMode,
compact,
countBy,
deepGet,
formatBytes,
@ -73,7 +74,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
full: [
'Datasource name',
'Availability',
'Segment load/drop queues',
'Availability detail',
'Total data size',
'Segment size',
'Segment granularity',
@ -89,7 +90,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'no-sql': [
'Datasource name',
'Availability',
'Segment load/drop queues',
'Availability detail',
'Total data size',
'Compaction',
'% Compacted',
@ -100,7 +101,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'no-proxy': [
'Datasource name',
'Availability',
'Segment load/drop queues',
'Availability detail',
'Total data size',
'Segment size',
'Segment granularity',
@ -150,7 +151,6 @@ const PERCENT_BRACES = [formatPercent(1)];
interface DatasourceQueryResultRow {
readonly datasource: string;
readonly num_segments: number;
readonly num_available_segments: number;
readonly num_segments_to_load: number;
readonly num_segments_to_drop: number;
readonly minute_aligned_segments: number;
@ -233,6 +233,12 @@ export interface DatasourcesViewState {
actions: BasicAction[];
}
interface DatasourceQuery {
capabilities: Capabilities;
hiddenColumns: LocalStorageBackedArray<string>;
showUnused: boolean;
}
export class DatasourcesView extends React.PureComponent<
DatasourcesViewProps,
DatasourcesViewState
@ -241,34 +247,49 @@ export class DatasourcesView extends React.PureComponent<
static FULLY_AVAILABLE_COLOR = '#57d500';
static PARTIALLY_AVAILABLE_COLOR = '#ffbf00';
static DATASOURCE_SQL = `SELECT
datasource,
COUNT(*) FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS num_segments,
COUNT(*) FILTER (WHERE is_available = 1 AND ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_available_segments,
COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load,
COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop,
COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments,
COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments,
COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments,
COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments,
COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments,
SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size,
SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size,
MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows,
AVG("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_rows,
MAX("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_rows,
SUM("num_rows") FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS total_rows,
CASE
WHEN SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) <> 0
THEN (
SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) /
SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0)
)
ELSE 0
END AS avg_row_size
static query(hiddenColumns: LocalStorageBackedArray<string>) {
const columns = compact(
[
hiddenColumns.exists('Datasource name') && `datasource`,
(hiddenColumns.exists('Availability') || hiddenColumns.exists('Segment granularity')) &&
`COUNT(*) FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS num_segments`,
(hiddenColumns.exists('Availability') || hiddenColumns.exists('Availability detail')) && [
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load`,
`COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop`,
],
hiddenColumns.exists('Total data size') &&
`SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size`,
hiddenColumns.exists('Segment size') && [
`MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows`,
`AVG("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_rows`,
`MAX("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_rows`,
],
hiddenColumns.exists('Segment granularity') && [
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments`,
],
hiddenColumns.exists('Total rows') &&
`SUM("num_rows") FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS total_rows`,
hiddenColumns.exists('Avg. row size') &&
`CASE WHEN SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) <> 0 THEN (SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) / SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0)) ELSE 0 END AS avg_row_size`,
hiddenColumns.exists('Replicated size') &&
`SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size`,
].flat(),
);
if (!columns.length) {
columns.push(`datasource`);
}
return `SELECT
${columns.join(',\n')}
FROM sys.segments
GROUP BY 1
ORDER BY 1`;
}
static formatRules(rules: Rule[]): string {
if (rules.length === 0) {
@ -280,7 +301,7 @@ ORDER BY 1`;
}
}
private datasourceQueryManager: QueryManager<Capabilities, DatasourcesAndDefaultRules>;
private datasourceQueryManager: QueryManager<DatasourceQuery, DatasourcesAndDefaultRules>;
private tiersQueryManager: QueryManager<Capabilities, string[]>;
constructor(props: DatasourcesViewProps, context: any) {
@ -312,10 +333,16 @@ ORDER BY 1`;
};
this.datasourceQueryManager = new QueryManager({
processQuery: async capabilities => {
processQuery: async (
{ capabilities, hiddenColumns, showUnused },
_cancelToken,
setIntermediateQuery,
) => {
let datasources: DatasourceQueryResultRow[];
if (capabilities.hasSql()) {
datasources = await queryDruidSql({ query: DatasourcesView.DATASOURCE_SQL });
const query = DatasourcesView.query(hiddenColumns);
setIntermediateQuery(query);
datasources = await queryDruidSql({ query });
} else if (capabilities.hasCoordinatorAccess()) {
const datasourcesResp = await Api.instance.get(
'/druid/coordinator/v1/datasources?simple',
@ -330,7 +357,6 @@ ORDER BY 1`;
const numSegments = availableSegments + segmentsToLoad;
return {
datasource: d.name,
num_available_segments: availableSegments,
num_segments: numSegments,
num_segments_to_load: segmentsToLoad,
num_segments_to_drop: 0,
@ -366,11 +392,9 @@ ORDER BY 1`;
const seen = countBy(datasources, x => x.datasource);
let unused: string[] = [];
if (this.state.showUnused) {
// Using 'includeDisabled' parameter for compatibility.
// Should be changed to 'includeUnused' in Druid 0.17
if (showUnused) {
const unusedResp = await Api.instance.get(
'/druid/coordinator/v1/metadata/datasources?includeDisabled',
'/druid/coordinator/v1/metadata/datasources?includeUnused',
);
unused = unusedResp.data.filter((d: string) => !seen[d]);
}
@ -442,9 +466,15 @@ ORDER BY 1`;
this.tiersQueryManager.rerunLastQuery(auto);
};
private fetchDatasourceData() {
const { capabilities } = this.props;
const { hiddenColumns, showUnused } = this.state;
this.datasourceQueryManager.runQuery({ capabilities, hiddenColumns, showUnused });
}
componentDidMount(): void {
const { capabilities } = this.props;
this.datasourceQueryManager.runQuery(capabilities);
this.fetchDatasourceData();
this.tiersQueryManager.runQuery(capabilities);
window.addEventListener('resize', this.handleResize);
}
@ -477,7 +507,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: undefined });
}}
onSuccess={() => {
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
}}
>
<p>
@ -510,7 +540,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: undefined });
}}
onSuccess={() => {
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
}}
>
<p>{`Are you sure you want to mark as used all non-overshadowed segments in '${datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn}'?`}</p>
@ -547,7 +577,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkSegmentsByIntervalIn: undefined });
}}
onSuccess={() => {
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
}}
>
<p>{`Please select the interval in which you want to mark segments as ${usedWord} in '${datasourceToMarkSegmentsByIntervalIn}'?`}</p>
@ -588,7 +618,7 @@ ORDER BY 1`;
this.setState({ killDatasource: undefined });
}}
onSuccess={() => {
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
}}
warningChecks={[
`I understand that this operation will delete all metadata about the unused segments of ${killDatasource} and removes them from deep storage.`,
@ -605,6 +635,7 @@ ORDER BY 1`;
renderBulkDatasourceActions() {
const { goToQuery, capabilities } = this.props;
const lastDatasourcesQuery = this.datasourceQueryManager.getLastIntermediateQuery();
return (
<MoreButton
@ -623,7 +654,11 @@ ORDER BY 1`;
<MenuItem
icon={IconNames.APPLICATION}
text="View SQL query for table"
onClick={() => goToQuery(DatasourcesView.DATASOURCE_SQL)}
disabled={!lastDatasourcesQuery}
onClick={() => {
if (!lastDatasourcesQuery) return;
goToQuery(lastDatasourcesQuery);
}}
/>
)}
<MenuItem
@ -680,7 +715,7 @@ ORDER BY 1`;
message: 'Retention rules submitted successfully',
intent: Intent.SUCCESS,
});
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
};
private editDefaultRules = () => {
@ -705,7 +740,7 @@ ORDER BY 1`;
try {
await Api.instance.post(`/druid/coordinator/v1/config/compaction`, compactionConfig);
this.setState({ compactionDialogOpenOn: undefined });
this.datasourceQueryManager.rerunLastQuery();
this.fetchDatasourceData();
} catch (e) {
AppToaster.show({
message: getDruidErrorMessage(e),
@ -728,9 +763,7 @@ ORDER BY 1`;
await Api.instance.delete(
`/druid/coordinator/v1/config/compaction/${Api.encodePath(datasource)}`,
);
this.setState({ compactionDialogOpenOn: undefined }, () =>
this.datasourceQueryManager.rerunLastQuery(),
);
this.setState({ compactionDialogOpenOn: undefined }, () => this.fetchDatasourceData());
} catch (e) {
AppToaster.show({
message: getDruidErrorMessage(e),
@ -743,10 +776,10 @@ ORDER BY 1`;
};
private toggleUnused(showUnused: boolean) {
if (!showUnused) {
this.datasourceQueryManager.rerunLastQuery();
}
this.setState({ showUnused: !showUnused });
this.setState({ showUnused: !showUnused }, () => {
if (showUnused) return;
this.fetchDatasourceData();
});
}
getDatasourceActions(
@ -979,18 +1012,11 @@ ORDER BY 1`;
{
Header: 'Availability',
show: hiddenColumns.exists('Availability'),
id: 'availability',
filterable: false,
minWidth: 200,
accessor: row => {
return {
num_available: row.num_available_segments,
num_total: row.num_segments,
};
},
Cell: ({ original }) => {
const { datasource, num_available_segments, num_segments, unused } = original;
accessor: 'num_segments',
Cell: ({ value: num_segments, original }) => {
const { datasource, unused, num_segments_to_load } = original;
if (unused) {
return (
<span>
@ -1005,7 +1031,9 @@ ORDER BY 1`;
{pluralIfNeeded(num_segments, 'segment')}
</a>
);
if (num_available_segments === num_segments) {
if (typeof num_segments_to_load !== 'number' || typeof num_segments !== 'number') {
return '-';
} else if (num_segments_to_load === 0) {
return (
<span>
<span style={{ color: DatasourcesView.FULLY_AVAILABLE_COLOR }}>
@ -1015,22 +1043,16 @@ ORDER BY 1`;
</span>
);
} else {
const numAvailableSegments = num_segments - num_segments_to_load;
const percentAvailable = (
Math.floor((num_available_segments / num_segments) * 1000) / 10
Math.floor((numAvailableSegments / num_segments) * 1000) / 10
).toFixed(1);
const missing = num_segments - num_available_segments;
const segmentsMissingEl = (
<a onClick={() => goToSegments(datasource, true)}>{`${pluralIfNeeded(
missing,
'segment',
)} unavailable`}</a>
);
return (
<span>
<span style={{ color: DatasourcesView.PARTIALLY_AVAILABLE_COLOR }}>
{num_available_segments ? '\u25cf' : '\u25cb'}&nbsp;
{numAvailableSegments ? '\u25cf' : '\u25cb'}&nbsp;
</span>
{percentAvailable}% available ({segmentsEl}, {segmentsMissingEl})
{percentAvailable}% available ({segmentsEl})
</span>
);
}
@ -1042,9 +1064,8 @@ ORDER BY 1`;
},
},
{
Header: twoLines('Segment load/drop', 'queues'),
show: hiddenColumns.exists('Segment load/drop queues'),
id: 'load-drop',
Header: twoLines('Availability', 'detail'),
show: hiddenColumns.exists('Availability detail'),
accessor: 'num_segments_to_load',
filterable: false,
minWidth: 100,
@ -1069,21 +1090,25 @@ ORDER BY 1`;
accessor: 'avg_segment_rows',
filterable: false,
width: 220,
Cell: ({ value, original }) => (
<>
<BracedText
text={formatSegmentRows(original.min_segment_rows)}
braces={minSegmentRowsValues}
/>{' '}
&nbsp;{' '}
<BracedText text={formatSegmentRows(value)} braces={avgSegmentRowsValues} />{' '}
&nbsp;{' '}
<BracedText
text={formatSegmentRows(original.max_segment_rows)}
braces={maxSegmentRowsValues}
/>
</>
),
Cell: ({ value, original }) => {
const { min_segment_rows, max_segment_rows } = original;
if (isNaN(value) || isNaN(min_segment_rows) || isNaN(max_segment_rows)) return '-';
return (
<>
<BracedText
text={formatSegmentRows(min_segment_rows)}
braces={minSegmentRowsValues}
/>{' '}
&nbsp;{' '}
<BracedText text={formatSegmentRows(value)} braces={avgSegmentRowsValues} />{' '}
&nbsp;{' '}
<BracedText
text={formatSegmentRows(max_segment_rows)}
braces={maxSegmentRowsValues}
/>
</>
);
},
},
{
Header: twoLines('Segment', 'granularity'),
@ -1093,24 +1118,32 @@ ORDER BY 1`;
filterable: false,
width: 100,
Cell: ({ original }) => {
const {
num_segments,
minute_aligned_segments,
hour_aligned_segments,
day_aligned_segments,
month_aligned_segments,
year_aligned_segments,
} = original;
const segmentGranularities: string[] = [];
if (!original.num_segments) return '-';
if (original.num_segments - original.minute_aligned_segments) {
if (!num_segments || isNaN(year_aligned_segments)) return '-';
if (num_segments - minute_aligned_segments) {
segmentGranularities.push('Sub minute');
}
if (original.minute_aligned_segments - original.hour_aligned_segments) {
if (minute_aligned_segments - hour_aligned_segments) {
segmentGranularities.push('Minute');
}
if (original.hour_aligned_segments - original.day_aligned_segments) {
if (hour_aligned_segments - day_aligned_segments) {
segmentGranularities.push('Hour');
}
if (original.day_aligned_segments - original.month_aligned_segments) {
if (day_aligned_segments - month_aligned_segments) {
segmentGranularities.push('Day');
}
if (original.month_aligned_segments - original.year_aligned_segments) {
if (month_aligned_segments - year_aligned_segments) {
segmentGranularities.push('Month');
}
if (original.year_aligned_segments) {
if (year_aligned_segments) {
segmentGranularities.push('Year');
}
return segmentGranularities.join(', ');
@ -1122,9 +1155,10 @@ ORDER BY 1`;
accessor: 'total_rows',
filterable: false,
width: 100,
Cell: ({ value }) => (
<BracedText text={formatTotalRows(value)} braces={totalRowsValues} />
),
Cell: ({ value }) => {
if (isNaN(value)) return '-';
return <BracedText text={formatTotalRows(value)} braces={totalRowsValues} />;
},
},
{
Header: twoLines('Avg. row size', '(bytes)'),
@ -1132,9 +1166,10 @@ ORDER BY 1`;
accessor: 'avg_row_size',
filterable: false,
width: 100,
Cell: ({ value }) => (
<BracedText text={formatAvgRowSize(value)} braces={avgRowSizeValues} />
),
Cell: ({ value }) => {
if (isNaN(value)) return '-';
return <BracedText text={formatAvgRowSize(value)} braces={avgRowSizeValues} />;
},
},
{
Header: twoLines('Replicated', 'size'),
@ -1142,9 +1177,12 @@ ORDER BY 1`;
accessor: 'replicated_size',
filterable: false,
width: 100,
Cell: ({ value }) => (
<BracedText text={formatReplicatedSize(value)} braces={replicatedSizeValues} />
),
Cell: ({ value }) => {
if (isNaN(value)) return '-';
return (
<BracedText text={formatReplicatedSize(value)} braces={replicatedSizeValues} />
);
},
},
{
Header: 'Compaction',
@ -1371,6 +1409,10 @@ ORDER BY 1`;
hiddenColumns: prevState.hiddenColumns.toggle(column),
}))
}
onClose={added => {
if (!added) return;
this.fetchDatasourceData();
}}
tableColumnsHidden={hiddenColumns.storedArray}
/>
</ViewControlBar>

15
web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
View File

@ -62,6 +62,7 @@ exports[`segments-view matches snapshot 1`] = `
]
}
onChange={[Function]}
onClose={[Function]}
tableColumnsHidden={Array []}
/>
</Memo(ViewControlBar)>
@ -125,7 +126,9 @@ exports[`segments-view matches snapshot 1`] = `
Object {
"Header": "Segment ID",
"accessor": "segment_id",
"filterable": true,
"show": true,
"sortable": true,
"width": 300,
},
Object {
@ -139,7 +142,9 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "Interval",
"accessor": "interval",
"defaultSortDesc": true,
"filterable": true,
"show": false,
"sortable": true,
"width": 120,
},
Object {
@ -147,7 +152,9 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "Start",
"accessor": "start",
"defaultSortDesc": true,
"filterable": true,
"show": true,
"sortable": true,
"width": 120,
},
Object {
@ -155,14 +162,18 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "End",
"accessor": "end",
"defaultSortDesc": true,
"filterable": true,
"show": true,
"sortable": true,
"width": 120,
},
Object {
"Header": "Version",
"accessor": "version",
"defaultSortDesc": true,
"filterable": true,
"show": true,
"sortable": true,
"width": 120,
},
Object {
@ -171,6 +182,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "time_span",
"filterable": true,
"show": true,
"sortable": true,
"width": 100,
},
Object {
@ -179,6 +191,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "partitioning",
"filterable": true,
"show": true,
"sortable": true,
"width": 100,
},
Object {
@ -186,6 +199,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "partition_num",
"filterable": false,
"show": true,
"sortable": true,
"width": 60,
},
Object {
@ -195,6 +209,7 @@ exports[`segments-view matches snapshot 1`] = `
"defaultSortDesc": true,
"filterable": false,
"show": true,
"sortable": true,
},
Object {
"Cell": [Function],

487
web-console/src/views/segments-view/segments-view.tsx
View File

@ -19,7 +19,6 @@
import { Button, ButtonGroup, Intent, Label, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { SqlExpression, SqlRef } from 'druid-query-toolkit';
import * as JSONBig from 'json-bigint-native';
import React from 'react';
import ReactTable, { Filter } from 'react-table';
@ -39,11 +38,13 @@ import { SegmentTableActionDialog } from '../../dialogs/segments-table-action-di
import { Api } from '../../singletons';
import {
addFilter,
booleanCustomTableFilter,
compact,
deepGet,
filterMap,
formatBytes,
formatInteger,
getNeedleAndMode,
LocalStorageKeys,
makeBooleanFilter,
queryDruidSql,
@ -124,6 +125,8 @@ interface TableState {
}
interface SegmentsQuery extends TableState {
hiddenColumns: LocalStorageBackedArray<string>;
capabilities: Capabilities;
groupByInterval: boolean;
}
@ -131,6 +134,7 @@ interface SegmentQueryResultRow {
datasource: string;
start: string;
end: string;
interval: string;
segment_id: string;
version: string;
time_span: string;
@ -147,7 +151,6 @@ interface SegmentQueryResultRow {
export interface SegmentsViewState {
segmentsState: QueryState<SegmentQueryResultRow[]>;
trimmedSegments?: SegmentQueryResultRow[];
segmentFilter: Filter[];
segmentTableActionDialogId?: string;
datasourceTableActionDialogId?: string;
@ -161,33 +164,74 @@ export interface SegmentsViewState {
export class SegmentsView extends React.PureComponent<SegmentsViewProps, SegmentsViewState> {
static PAGE_SIZE = 25;
static WITH_QUERY = `WITH s AS (
SELECT
"segment_id", "datasource", "start", "end", "size", "version",
CASE
WHEN "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z' THEN 'Year'
WHEN "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z' THEN 'Month'
WHEN "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z' THEN 'Day'
WHEN "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z' THEN 'Hour'
WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
ELSE 'Sub minute'
END AS "time_span",
CASE
WHEN "shard_spec" LIKE '%"type":"numbered"%' THEN 'dynamic'
WHEN "shard_spec" LIKE '%"type":"hashed"%' THEN 'hashed'
WHEN "shard_spec" LIKE '%"type":"single"%' THEN 'single_dim'
WHEN "shard_spec" LIKE '%"type":"none"%' THEN 'none'
WHEN "shard_spec" LIKE '%"type":"linear"%' THEN 'linear'
WHEN "shard_spec" LIKE '%"type":"numbered_overwrite"%' THEN 'numbered_overwrite'
ELSE '-'
END AS "partitioning",
"partition_num", "num_replicas", "num_rows",
"is_published", "is_available", "is_realtime", "is_overshadowed"
FROM sys.segments
)`;
static baseQuery(hiddenColumns: LocalStorageBackedArray<string>) {
const columns = compact([
hiddenColumns.exists('Segment ID') && `"segment_id"`,
hiddenColumns.exists('Datasource') && `"datasource"`,
hiddenColumns.exists('Start') && `"start"`,
hiddenColumns.exists('End') && `"end"`,
hiddenColumns.exists('Version') && `"version"`,
hiddenColumns.exists('Time span') &&
`CASE
WHEN "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z' THEN 'Year'
WHEN "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z' THEN 'Month'
WHEN "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z' THEN 'Day'
WHEN "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z' THEN 'Hour'
WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
ELSE 'Sub minute'
END AS "time_span"`,
hiddenColumns.exists('Partitioning') &&
`CASE
WHEN "shard_spec" LIKE '%"type":"numbered"%' THEN 'dynamic'
WHEN "shard_spec" LIKE '%"type":"hashed"%' THEN 'hashed'
WHEN "shard_spec" LIKE '%"type":"single"%' THEN 'single_dim'
WHEN "shard_spec" LIKE '%"type":"none"%' THEN 'none'
WHEN "shard_spec" LIKE '%"type":"linear"%' THEN 'linear'
WHEN "shard_spec" LIKE '%"type":"numbered_overwrite"%' THEN 'numbered_overwrite'
ELSE '-'
END AS "partitioning"`,
hiddenColumns.exists('Partition') && `"partition_num"`,
hiddenColumns.exists('Size') && `"size"`,
hiddenColumns.exists('Num rows') && `"num_rows"`,
hiddenColumns.exists('Replicas') && `"num_replicas"`,
hiddenColumns.exists('Is published') && `"is_published"`,
hiddenColumns.exists('Is available') && `"is_available"`,
hiddenColumns.exists('Is realtime') && `"is_realtime"`,
hiddenColumns.exists('Is overshadowed') && `"is_overshadowed"`,
]);
private segmentsSqlQueryManager: QueryManager<SegmentsQuery, SegmentQueryResultRow[]>;
private segmentsNoSqlQueryManager: QueryManager<null, SegmentQueryResultRow[]>;
if (!columns.length) {
columns.push(`"segment_id"`);
}
return `WITH s AS (SELECT\n${columns.join(',\n')}\nFROM sys.segments)`;
}
static computeTimeSpan(start: string, end: string): string {
if (start.endsWith('-01-01T00:00:00.000Z') && end.endsWith('-01-01T00:00:00.000Z')) {
return 'Year';
}
if (start.endsWith('-01T00:00:00.000Z') && end.endsWith('-01T00:00:00.000Z')) {
return 'Month';
}
if (start.endsWith('T00:00:00.000Z') && end.endsWith('T00:00:00.000Z')) {
return 'Day';
}
if (start.endsWith(':00:00.000Z') && end.endsWith(':00:00.000Z')) {
return 'Hour';
}
if (start.endsWith(':00.000Z') && end.endsWith(':00.000Z')) {
return 'Minute';
}
return 'Sub minute';
}
private segmentsQueryManager: QueryManager<SegmentsQuery, SegmentQueryResultRow[]>;
private lastTableState: TableState | undefined;
@ -208,194 +252,189 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
groupByInterval: false,
};
this.segmentsSqlQueryManager = new QueryManager({
this.segmentsQueryManager = new QueryManager({
debounceIdle: 500,
processQuery: async (query: SegmentsQuery, _cancelToken, setIntermediateQuery) => {
const whereParts = filterMap(query.filtered, (f: Filter) => {
if (f.id.startsWith('is_')) {
if (f.value === 'all') return;
return SqlRef.columnWithQuotes(f.id).equal(f.value === 'true' ? 1 : 0);
const {
page,
pageSize,
filtered,
sorted,
hiddenColumns,
capabilities,
groupByInterval,
} = query;
if (capabilities.hasSql()) {
const whereParts = filterMap(filtered, (f: Filter) => {
if (f.id.startsWith('is_')) {
if (f.value === 'all') return;
return SqlRef.columnWithQuotes(f.id).equal(f.value === 'true' ? 1 : 0);
} else {
return sqlQueryCustomTableFilter(f);
}
});
let queryParts: string[];
let whereClause = '';
if (whereParts.length) {
whereClause = SqlExpression.and(...whereParts).toString();
}
if (groupByInterval) {
const innerQuery = compact([
`SELECT "start" || '/' || "end" AS "interval"`,
`FROM sys.segments`,
whereClause ? `WHERE ${whereClause}` : undefined,
`GROUP BY 1`,
`ORDER BY 1 DESC`,
`LIMIT ${pageSize}`,
page ? `OFFSET ${page * pageSize}` : undefined,
]).join('\n');
const intervals: string = (await queryDruidSql({ query: innerQuery }))
.map(row => `'${row.interval}'`)
.join(', ');
queryParts = compact([
SegmentsView.baseQuery(hiddenColumns),
`SELECT "start" || '/' || "end" AS "interval", *`,
`FROM s`,
`WHERE`,
intervals ? ` ("start" || '/' || "end") IN (${intervals})` : 'FALSE',
whereClause ? ` AND ${whereClause}` : '',
]);
if (sorted.length) {
queryParts.push(
'ORDER BY ' +
sorted
.map((sort: any) => `${SqlRef.column(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
.join(', '),
);
}
queryParts.push(`LIMIT ${pageSize * 1000}`);
} else {
return sqlQueryCustomTableFilter(f);
queryParts = [SegmentsView.baseQuery(hiddenColumns), `SELECT *`, `FROM s`];
if (whereClause) {
queryParts.push(`WHERE ${whereClause}`);
}
if (sorted.length) {
queryParts.push(
'ORDER BY ' +
sorted
.map((sort: any) => `${SqlRef.column(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
.join(', '),
);
}
queryParts.push(`LIMIT ${pageSize}`);
if (page) {
queryParts.push(`OFFSET ${page * pageSize}`);
}
}
});
const sqlQuery = queryParts.join('\n');
setIntermediateQuery(sqlQuery);
return await queryDruidSql({ query: sqlQuery });
} else if (capabilities.hasCoordinatorAccess()) {
let datasourceList: string[] = (await Api.instance.get(
'/druid/coordinator/v1/metadata/datasources',
)).data;
let queryParts: string[];
let whereClause = '';
if (whereParts.length) {
whereClause = SqlExpression.and(...whereParts).toString();
}
if (query.groupByInterval) {
const innerQuery = compact([
`SELECT "start" || '/' || "end" AS "interval"`,
`FROM sys.segments`,
whereClause ? `WHERE ${whereClause}` : undefined,
`GROUP BY 1`,
`ORDER BY 1 DESC`,
`LIMIT ${query.pageSize}`,
query.page ? `OFFSET ${query.page * query.pageSize}` : undefined,
]).join('\n');
const intervals: string = (await queryDruidSql({ query: innerQuery }))
.map(row => `'${row.interval}'`)
.join(', ');
queryParts = compact([
SegmentsView.WITH_QUERY,
`SELECT "start" || '/' || "end" AS "interval", *`,
`FROM s`,
`WHERE`,
intervals ? ` ("start" || '/' || "end") IN (${intervals})` : 'FALSE',
whereClause ? ` AND ${whereClause}` : '',
]);
if (query.sorted.length) {
queryParts.push(
'ORDER BY ' +
query.sorted
.map((sort: any) => `${JSONBig.stringify(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
.join(', '),
const datasourceFilter = filtered.find(({ id }) => id === 'datasource');
if (datasourceFilter) {
datasourceList = datasourceList.filter(datasource =>
booleanCustomTableFilter(datasourceFilter, datasource),
);
}
queryParts.push(`LIMIT ${query.pageSize * 1000}`);
} else {
queryParts = [SegmentsView.WITH_QUERY, `SELECT *`, `FROM s`];
if (whereClause) {
queryParts.push(`WHERE ${whereClause}`);
}
if (query.sorted.length) {
queryParts.push(
'ORDER BY ' +
query.sorted
.map((sort: any) => `${JSONBig.stringify(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
.join(', '),
if (sorted.length && sorted[0].id === 'datasource') {
datasourceList.sort(
sorted[0].desc ? (d1, d2) => d1.localeCompare(d2) : (d1, d2) => d2.localeCompare(d1),
);
}
queryParts.push(`LIMIT ${query.pageSize}`);
const maxResults = (page + 1) * pageSize;
let results: SegmentQueryResultRow[] = [];
if (query.page) {
queryParts.push(`OFFSET ${query.page * query.pageSize}`);
}
}
const sqlQuery = queryParts.join('\n');
setIntermediateQuery(sqlQuery);
return await queryDruidSql({ query: sqlQuery });
},
onStateChange: segmentsState => {
this.setState({
segmentsState,
});
},
});
this.segmentsNoSqlQueryManager = new QueryManager({
processQuery: async () => {
const datasourceList = (await Api.instance.get(
'/druid/coordinator/v1/metadata/datasources',
)).data;
const nestedResults: SegmentQueryResultRow[][] = await Promise.all(
datasourceList.map(async (d: string) => {
const n = Math.min(datasourceList.length, maxResults);
for (let i = 0; i < n && results.length < maxResults; i++) {
const segments = (await Api.instance.get(
`/druid/coordinator/v1/datasources/${Api.encodePath(d)}?full`,
`/druid/coordinator/v1/datasources/${Api.encodePath(datasourceList[i])}?full`,
)).data.segments;
if (!Array.isArray(segments)) continue;
return segments.map(
(segment: any): SegmentQueryResultRow => {
return {
segment_id: segment.identifier,
datasource: segment.dataSource,
start: segment.interval.split('/')[0],
end: segment.interval.split('/')[1],
version: segment.version,
time_span: '-',
partitioning: '-',
partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0,
size: segment.size,
num_rows: -1,
num_replicas: -1,
is_available: -1,
is_published: -1,
is_realtime: -1,
is_overshadowed: -1,
};
},
);
}),
);
let segmentQueryResultRows: SegmentQueryResultRow[] = segments.map((segment: any) => {
const [start, end] = segment.interval.split('/');
return {
segment_id: segment.identifier,
datasource: segment.dataSource,
start,
end,
interval: segment.interval,
version: segment.version,
time_span: SegmentsView.computeTimeSpan(start, end),
partitioning: deepGet(segment, 'shardSpec.type') || '-',
partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0,
size: segment.size,
num_rows: -1,
num_replicas: -1,
is_available: -1,
is_published: -1,
is_realtime: -1,
is_overshadowed: -1,
};
});
return nestedResults.flat().sort((d1, d2) => {
return d2.start.localeCompare(d1.start);
});
if (filtered.length) {
segmentQueryResultRows = segmentQueryResultRows.filter((d: SegmentQueryResultRow) => {
return filtered.every(filter => {
return booleanCustomTableFilter(
filter,
d[filter.id as keyof SegmentQueryResultRow],
);
});
});
}
results = results.concat(segmentQueryResultRows);
}
return results.slice(page * pageSize, maxResults);
} else {
throw new Error('must have SQL or coordinator access to load this view');
}
},
onStateChange: segmentsState => {
this.setState({
trimmedSegments: segmentsState.data
? segmentsState.data.slice(0, SegmentsView.PAGE_SIZE)
: undefined,
segmentsState,
});
},
});
}
componentDidMount(): void {
const { capabilities } = this.props;
if (!capabilities.hasSql() && capabilities.hasCoordinatorAccess()) {
this.segmentsNoSqlQueryManager.runQuery(null);
}
}
componentWillUnmount(): void {
this.segmentsSqlQueryManager.terminate();
this.segmentsNoSqlQueryManager.terminate();
this.segmentsQueryManager.terminate();
}
private fetchData = (groupByInterval: boolean, tableState?: TableState) => {
const { capabilities } = this.props;
const { hiddenColumns } = this.state;
if (tableState) this.lastTableState = tableState;
const { page, pageSize, filtered, sorted } = this.lastTableState!;
this.segmentsSqlQueryManager.runQuery({
this.segmentsQueryManager.runQuery({
page,
pageSize,
filtered,
sorted,
groupByInterval: groupByInterval,
});
};
private fetchClientSideData = (tableState?: TableState) => {
if (tableState) this.lastTableState = tableState;
const { page, pageSize, filtered, sorted } = this.lastTableState!;
this.setState(state => {
const allSegments = state.segmentsState.data;
if (!allSegments) return {};
const sortKey = sorted[0].id as keyof SegmentQueryResultRow;
const sortDesc = sorted[0].desc;
return {
trimmedSegments: allSegments
.filter(d => {
return filtered.every((f: any) => {
return String(d[f.id as keyof SegmentQueryResultRow]).includes(f.value);
});
})
.sort((d1, d2) => {
const v1 = d1[sortKey] as any;
const v2 = d2[sortKey] as any;
if (typeof v1 === 'string') {
return sortDesc ? v2.localeCompare(v1) : v1.localeCompare(v2);
} else {
return sortDesc ? v2 - v1 : v1 - v2;
}
})
.slice(page * pageSize, (page + 1) * pageSize),
};
hiddenColumns,
capabilities,
groupByInterval,
});
};
@ -411,16 +450,10 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
}
renderSegmentsTable() {
const {
segmentsState,
trimmedSegments,
segmentFilter,
hiddenColumns,
groupByInterval,
} = this.state;
const { segmentsState, segmentFilter, hiddenColumns, groupByInterval } = this.state;
const { capabilities } = this.props;
const segments = trimmedSegments || segmentsState.data || [];
const segments = segmentsState.data || [];
const sizeValues = segments.map(d => formatBytes(d.size)).concat('(realtime)');
@ -443,6 +476,15 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
};
};
const hasSql = capabilities.hasSql();
// Only allow filtering of columns other than datasource if in SQL mode or we are filtering on an exact datasource
const allowGeneralFilter =
hasSql ||
segmentFilter.some(
filter => filter.id === 'datasource' && getNeedleAndMode(filter).mode === 'exact',
);
return (
<ReactTable
data={segments}
@ -452,16 +494,12 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
manual
filterable
filtered={segmentFilter}
defaultSorted={[{ id: 'start', desc: true }]}
defaultSorted={[hasSql ? { id: 'start', desc: true } : { id: 'datasource', desc: false }]}
onFilteredChange={filtered => {
this.setState({ segmentFilter: filtered });
}}
onFetchData={tableState => {
if (capabilities.hasSql()) {
this.fetchData(groupByInterval, tableState);
} else if (capabilities.hasCoordinatorAccess()) {
this.fetchClientSideData(tableState);
}
this.fetchData(groupByInterval, tableState);
}}
showPageJump={false}
ofText=""
@ -472,6 +510,8 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: hiddenColumns.exists('Segment ID'),
accessor: 'segment_id',
width: 300,
sortable: hasSql,
filterable: allowGeneralFilter,
},
{
Header: 'Datasource',
@ -484,7 +524,9 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: groupByInterval,
accessor: 'interval',
width: 120,
sortable: hasSql,
defaultSortDesc: true,
filterable: allowGeneralFilter,
Cell: renderFilterableCell('interval'),
},
{
@ -492,38 +534,46 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: hiddenColumns.exists('Start'),
accessor: 'start',
width: 120,
sortable: hasSql,
defaultSortDesc: true,
filterable: allowGeneralFilter,
Cell: renderFilterableCell('start'),
},
{
Header: 'End',
show: hiddenColumns.exists('End'),
accessor: 'end',
defaultSortDesc: true,
width: 120,
sortable: hasSql,
defaultSortDesc: true,
filterable: allowGeneralFilter,
Cell: renderFilterableCell('end'),
},
{
Header: 'Version',
show: hiddenColumns.exists('Version'),
accessor: 'version',
defaultSortDesc: true,
width: 120,
sortable: hasSql,
defaultSortDesc: true,
filterable: allowGeneralFilter,
},
{
Header: 'Time span',
show: capabilities.hasSql() && hiddenColumns.exists('Time span'),
show: hiddenColumns.exists('Time span'),
accessor: 'time_span',
width: 100,
filterable: true,
sortable: hasSql,
filterable: allowGeneralFilter,
Cell: renderFilterableCell('time_span'),
},
{
Header: 'Partitioning',
show: capabilities.hasSql() && hiddenColumns.exists('Partitioning'),
show: hiddenColumns.exists('Partitioning'),
accessor: 'partitioning',
width: 100,
filterable: true,
sortable: hasSql,
filterable: allowGeneralFilter,
Cell: renderFilterableCell('partitioning'),
},
{
@ -532,12 +582,14 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
accessor: 'partition_num',
width: 60,
filterable: false,
sortable: hasSql,
},
{
Header: 'Size',
show: hiddenColumns.exists('Size'),
accessor: 'size',
filterable: false,
sortable: hasSql,
defaultSortDesc: true,
Cell: row => (
<BracedText
@ -552,7 +604,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Num rows',
show: capabilities.hasSql() && hiddenColumns.exists('Num rows'),
show: hasSql && hiddenColumns.exists('Num rows'),
accessor: 'num_rows',
filterable: false,
defaultSortDesc: true,
@ -565,7 +617,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Replicas',
show: capabilities.hasSql() && hiddenColumns.exists('Replicas'),
show: hasSql && hiddenColumns.exists('Replicas'),
accessor: 'num_replicas',
width: 60,
filterable: false,
@ -573,28 +625,28 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Is published',
show: capabilities.hasSql() && hiddenColumns.exists('Is published'),
show: hasSql && hiddenColumns.exists('Is published'),
id: 'is_published',
accessor: row => String(Boolean(row.is_published)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is realtime',
show: capabilities.hasSql() && hiddenColumns.exists('Is realtime'),
show: hasSql && hiddenColumns.exists('Is realtime'),
id: 'is_realtime',
accessor: row => String(Boolean(row.is_realtime)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is available',
show: capabilities.hasSql() && hiddenColumns.exists('Is available'),
show: hasSql && hiddenColumns.exists('Is available'),
id: 'is_available',
accessor: row => String(Boolean(row.is_available)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is overshadowed',
show: capabilities.hasSql() && hiddenColumns.exists('Is overshadowed'),
show: hasSql && hiddenColumns.exists('Is overshadowed'),
id: 'is_overshadowed',
accessor: row => String(Boolean(row.is_overshadowed)),
Filter: makeBooleanFilter(),
@ -654,8 +706,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
this.setState({ terminateSegmentId: undefined });
}}
onSuccess={() => {
this.segmentsNoSqlQueryManager.rerunLastQuery();
this.segmentsSqlQueryManager.rerunLastQuery();
this.segmentsQueryManager.rerunLastQuery();
}}
>
<p>{`Are you sure you want to drop segment '${terminateSegmentId}'?`}</p>
@ -666,7 +717,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
renderBulkSegmentsActions() {
const { goToQuery, capabilities } = this.props;
const lastSegmentsQuery = this.segmentsSqlQueryManager.getLastIntermediateQuery();
const lastSegmentsQuery = this.segmentsQueryManager.getLastIntermediateQuery();
return (
<MoreButton>
@ -700,11 +751,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
<div className="segments-view app-view">
<ViewControlBar label="Segments">
<RefreshButton
onRefresh={auto =>
capabilities.hasSql()
? this.segmentsSqlQueryManager.rerunLastQuery(auto)
: this.segmentsNoSqlQueryManager.rerunLastQuery(auto)
}
onRefresh={auto => this.segmentsQueryManager.rerunLastQuery(auto)}
localStorageKey={LocalStorageKeys.SEGMENTS_REFRESH_RATE}
/>
<Label>Group by</Label>
@ -713,11 +760,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
active={!groupByInterval}
onClick={() => {
this.setState({ groupByInterval: false });
if (capabilities.hasSql()) {
this.fetchData(false);
} else {
this.fetchClientSideData();
}
this.fetchData(false);
}}
>
None
@ -740,6 +783,10 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
hiddenColumns: prevState.hiddenColumns.toggle(column),
}))
}
onClose={added => {
if (!added) return;
this.fetchData(groupByInterval);
}}
tableColumnsHidden={hiddenColumns.storedArray}
/>
</ViewControlBar>