Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: default max workers to cluster capacity and simplify live reports (#13577) 

* step

* better capacity

* start with capacity

* more compressed stats display

* better rule editor

* fix SQL data loader also

* update snapshots

* new line

* better formatting
This commit is contained in:
Vadim Ogievetsky 2022-12-16 15:13:32 -08:00 committed by GitHub
parent 639decdf2e
commit e23abc710a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 760 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
web-console/src/components/braced-text/braced-text.tsx
View File

@ -30,6 +30,7 @@ export interface BracedTextProps {
braces: string[];
padFractionalPart?: boolean;
unselectableThousandsSeparator?: boolean;
title?: string;
}
export function findMostNumbers(strings: string[]): string {
@ -82,7 +83,8 @@ function hideThousandsSeparator(text: string) {
}
export const BracedText = React.memo(function BracedText(props: BracedTextProps) {
const { className, text, braces, padFractionalPart, unselectableThousandsSeparator } = props;
const { className, text, braces, padFractionalPart, unselectableThousandsSeparator, title } =
props;
let effectiveBraces = braces.concat(text);
@ -112,7 +114,7 @@ export const BracedText = React.memo(function BracedText(props: BracedTextProps)
}
return (
<span className={classNames('braced-text', className)}>
<span className={classNames('braced-text', className)} title={title}>
<span className="brace-text">{findMostNumbers(effectiveBraces)}</span>
<span className="real-text">
{unselectableThousandsSeparator ? hideThousandsSeparator(text) : text}

1
web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
View File

@ -181,6 +181,7 @@ exports[`HeaderBar matches snapshot 1`] = `
<Memo(RestrictedMode)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,

2
web-console/src/components/header-bar/header-bar.spec.tsx
View File

@ -19,7 +19,7 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { HeaderBar } from './header-bar';

2
web-console/src/components/header-bar/header-bar.tsx
View File

@ -40,9 +40,9 @@ import {
DoctorDialog,
OverlordDynamicConfigDialog,
} from '../../dialogs';
import { Capabilities } from '../../helpers';
import { getLink } from '../../links';
import {
Capabilities,
localStorageGetJson,
LocalStorageKeys,
localStorageRemove,

192
web-console/src/components/rule-editor/__snapshots__/rule-editor.spec.tsx.snap
View File

@ -14,7 +14,7 @@ exports[`RuleEditor matches snapshot no tier in rule 1`] = `
<span
class="bp4-button-text"
>
loadForever
loadForever(1x)
</span>
<span
aria-hidden="true"
@ -166,6 +166,170 @@ exports[`RuleEditor matches snapshot no tier in rule 1`] = `
<div
class="bp4-form-content"
>
<div
class="bp4-control-group"
>
<button
class="bp4-button bp4-minimal"
style="pointer-events: none;"
type="button"
>
<span
class="bp4-button-text"
>
Replicants:
</span>
</button>
<div
class="bp4-control-group bp4-numeric-input"
>
<div
class="bp4-input-group"
>
<input
autocomplete="off"
class="bp4-input"
max="256"
min="0"
type="text"
value="1"
/>
</div>
<div
class="bp4-button-group bp4-vertical bp4-fixed"
>
<button
aria-label="increment"
class="bp4-button"
type="button"
>
<span
aria-hidden="true"
class="bp4-icon bp4-icon-chevron-up"
icon="chevron-up"
tabindex="0"
>
<svg
data-icon="chevron-up"
height="16"
viewBox="0 0 16 16"
width="16"
>
<path
d="M12.71 9.29l-4-4C8.53 5.11 8.28 5 8 5s-.53.11-.71.29l-4 4a1.003 1.003 0 001.42 1.42L8 7.41l3.29 3.29c.18.19.43.3.71.3a1.003 1.003 0 00.71-1.71z"
fill-rule="evenodd"
/>
</svg>
</span>
</button>
<button
aria-label="decrement"
class="bp4-button"
type="button"
>
<span
aria-hidden="true"
class="bp4-icon bp4-icon-chevron-down"
icon="chevron-down"
tabindex="0"
>
<svg
data-icon="chevron-down"
height="16"
viewBox="0 0 16 16"
width="16"
>
<path
d="M12 5c-.28 0-.53.11-.71.29L8 8.59l-3.29-3.3a1.003 1.003 0 00-1.42 1.42l4 4c.18.18.43.29.71.29s.53-.11.71-.29l4-4A1.003 1.003 0 0012 5z"
fill-rule="evenodd"
/>
</svg>
</span>
</button>
</div>
</div>
<button
class="bp4-button bp4-minimal"
style="pointer-events: none;"
type="button"
>
<span
class="bp4-button-text"
>
Tier:
</span>
</button>
<div
class="bp4-html-select bp4-fill"
>
<select>
<option
value="test1"
>
test1
</option>
<option
value="test"
>
test
</option>
<option
value="test"
>
test
</option>
<option
value="test"
>
test
</option>
</select>
<span
class="bp4-icon bp4-icon-double-caret-vertical"
icon="double-caret-vertical"
>
<svg
data-icon="double-caret-vertical"
height="16"
viewBox="0 0 16 16"
width="16"
>
<desc>
Open dropdown
</desc>
<path
d="M5 7h6a1.003 1.003 0 00.71-1.71l-3-3C8.53 2.11 8.28 2 8 2s-.53.11-.71.29l-3 3A1.003 1.003 0 005 7zm6 2H5a1.003 1.003 0 00-.71 1.71l3 3c.18.18.43.29.71.29s.53-.11.71-.29l3-3A1.003 1.003 0 0011 9z"
fill-rule="evenodd"
/>
</svg>
</span>
</div>
<button
class="bp4-button bp4-disabled"
disabled=""
tabindex="-1"
type="button"
>
<span
aria-hidden="true"
class="bp4-icon bp4-icon-trash"
icon="trash"
tabindex="0"
>
<svg
data-icon="trash"
height="16"
viewBox="0 0 16 16"
width="16"
>
<path
d="M14.49 3.99h-13c-.28 0-.5.22-.5.5s.22.5.5.5h.5v10c0 .55.45 1 1 1h10c.55 0 1-.45 1-1v-10h.5c.28 0 .5-.22.5-.5s-.22-.5-.5-.5zm-8.5 9c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm3 0c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm3 0c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm2-12h-4c0-.55-.45-1-1-1h-2c-.55 0-1 .45-1 1h-4c-.55 0-1 .45-1 1v1h14v-1c0-.55-.45-1-1-1z"
fill-rule="evenodd"
/>
</svg>
</span>
</button>
</div>
<div
class="bp4-form-group right"
>
@ -374,7 +538,7 @@ exports[`RuleEditor matches snapshot with broadcast rule 1`] = `
class="bp4-input"
placeholder="2010-01-01/2020-01-01"
type="text"
value=""
value="2010-01-01/2015-01-01"
/>
</div>
</div>
@ -400,7 +564,7 @@ exports[`RuleEditor matches snapshot with existing tier and non existing tier in
<span
class="bp4-button-text"
>
loadByInterval(2010-01-01/2015-01-01)
loadByInterval(2010-01-01/2015-01-01, 3x)
</span>
<span
aria-hidden="true"
@ -550,7 +714,7 @@ exports[`RuleEditor matches snapshot with existing tier and non existing tier in
class="bp4-input"
placeholder="2010-01-01/2020-01-01"
type="text"
value=""
value="2010-01-01/2015-01-01"
/>
</div>
</div>
@ -586,7 +750,7 @@ exports[`RuleEditor matches snapshot with existing tier and non existing tier in
autocomplete="off"
class="bp4-input"
max="256"
min="1"
min="0"
type="text"
value="1"
/>
@ -620,9 +784,7 @@ exports[`RuleEditor matches snapshot with existing tier and non existing tier in
</button>
<button
aria-label="decrement"
class="bp4-button bp4-disabled"
disabled=""
tabindex="-1"
class="bp4-button"
type="button"
>
<span
@ -745,7 +907,7 @@ exports[`RuleEditor matches snapshot with existing tier and non existing tier in
autocomplete="off"
class="bp4-input"
max="256"
min="1"
min="0"
type="text"
value="2"
/>
@ -936,7 +1098,7 @@ exports[`RuleEditor matches snapshot with existing tier in rule 1`] = `
<span
class="bp4-button-text"
>
loadByInterval(2010-01-01/2015-01-01)
loadByInterval(2010-01-01/2015-01-01, 2x)
</span>
<span
aria-hidden="true"
@ -1086,7 +1248,7 @@ exports[`RuleEditor matches snapshot with existing tier in rule 1`] = `
class="bp4-input"
placeholder="2010-01-01/2020-01-01"
type="text"
value=""
value="2010-01-01/2015-01-01"
/>
</div>
</div>
@ -1122,7 +1284,7 @@ exports[`RuleEditor matches snapshot with existing tier in rule 1`] = `
autocomplete="off"
class="bp4-input"
max="256"
min="1"
min="0"
type="text"
value="2"
/>
@ -1315,7 +1477,7 @@ exports[`RuleEditor matches snapshot with non existing tier in rule 1`] = `
<span
class="bp4-button-text"
>
loadByInterval(2010-01-01/2015-01-01)
loadByInterval(2010-01-01/2015-01-01, 2x)
</span>
<span
aria-hidden="true"
@ -1465,7 +1627,7 @@ exports[`RuleEditor matches snapshot with non existing tier in rule 1`] = `
class="bp4-input"
placeholder="2010-01-01/2020-01-01"
type="text"
value=""
value="2010-01-01/2015-01-01"
/>
</div>
</div>
@ -1501,7 +1663,7 @@ exports[`RuleEditor matches snapshot with non existing tier in rule 1`] = `
autocomplete="off"
class="bp4-input"
max="256"
min="1"
min="0"
type="text"
value="2"
/>

10
web-console/src/components/rule-editor/rule-editor.spec.tsx
View File

@ -25,7 +25,7 @@ describe('RuleEditor', () => {
it('matches snapshot no tier in rule', () => {
const ruleEditor = (
<RuleEditor
rule={{ type: 'loadForever' }}
rule={{ type: 'loadForever', tieredReplicants: { test1: 1 } }}
tiers={['test', 'test', 'test']}
onChange={() => {}}
onDelete={() => {}}
@ -42,7 +42,7 @@ describe('RuleEditor', () => {
<RuleEditor
rule={{
type: 'loadByInterval',
period: '2010-01-01/2015-01-01',
interval: '2010-01-01/2015-01-01',
tieredReplicants: { nonexist: 2 },
}}
tiers={['test1', 'test2', 'test3']}
@ -61,7 +61,7 @@ describe('RuleEditor', () => {
<RuleEditor
rule={{
type: 'loadByInterval',
period: '2010-01-01/2015-01-01',
interval: '2010-01-01/2015-01-01',
tieredReplicants: { test1: 2 },
}}
tiers={['test1', 'test2', 'test3']}
@ -80,7 +80,7 @@ describe('RuleEditor', () => {
<RuleEditor
rule={{
type: 'loadByInterval',
period: '2010-01-01/2015-01-01',
interval: '2010-01-01/2015-01-01',
tieredReplicants: {
test1: 2,
nonexist: 1,
@ -102,7 +102,7 @@ describe('RuleEditor', () => {
<RuleEditor
rule={{
type: 'broadcastByInterval',
period: '2010-01-01/2015-01-01',
interval: '2010-01-01/2015-01-01',
}}
tiers={[]}
onChange={() => {}}

2
web-console/src/components/rule-editor/rule-editor.tsx
View File

@ -91,7 +91,7 @@ export const RuleEditor = React.memo(function RuleEditor(props: RuleEditorProps)
if (isNaN(v)) return;
onChange(RuleUtil.addTieredReplicant(rule, tier, v));
}}
min={1}
min={0}
max={256}
/>
<Button minimal style={{ pointerEvents: 'none' }}>

2
web-console/src/components/segment-timeline/segment-timeline.spec.tsx
View File

@ -20,7 +20,7 @@ import { render } from '@testing-library/react';
import { sane } from 'druid-query-toolkit';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { SegmentTimeline } from './segment-timeline';

2
web-console/src/components/segment-timeline/segment-timeline.tsx
View File

@ -28,9 +28,9 @@ import { AxisScale } from 'd3-axis';
import { scaleLinear, scaleUtc } from 'd3-scale';
import React from 'react';
import { Capabilities } from '../../helpers';
import { Api } from '../../singletons';
import {
Capabilities,
ceilToUtcDay,
formatBytes,
formatInteger,

25
web-console/src/console-application.tsx
View File

@ -25,8 +25,9 @@ import { HashRouter, Route, Switch } from 'react-router-dom';
import { HeaderActiveTab, HeaderBar, Loader } from './components';
import { DruidEngine, QueryWithContext } from './druid-models';
import { Capabilities } from './helpers';
import { AppToaster } from './singletons';
import { Capabilities, QueryManager } from './utils';
import { localStorageGetJson, LocalStorageKeys, QueryManager } from './utils';
import {
DatasourcesView,
HomeView,
@ -89,9 +90,17 @@ export class ConsoleApplication extends React.PureComponent<
this.capabilitiesQueryManager = new QueryManager({
processQuery: async () => {
const capabilities = await Capabilities.detectCapabilities();
if (!capabilities) ConsoleApplication.shownServiceNotification();
return capabilities || Capabilities.FULL;
const capabilitiesOverride = localStorageGetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE);
const capabilities = capabilitiesOverride
? new Capabilities(capabilitiesOverride)
: await Capabilities.detectCapabilities();
if (!capabilities) {
ConsoleApplication.shownServiceNotification();
return Capabilities.FULL;
}
return await Capabilities.detectCapacity(capabilities);
},
onStateChange: ({ data, loading }) => {
this.setState({
@ -259,6 +268,7 @@ export class ConsoleApplication extends React.PureComponent<
return this.wrapInViewContainer(
'workbench',
<WorkbenchView
capabilities={capabilities}
tabId={p.match.params.tabId}
onTabChange={newTabId => {
location.hash = `#workbench/${newTabId}`;
@ -275,9 +285,14 @@ export class ConsoleApplication extends React.PureComponent<
};
private readonly wrappedSqlDataLoaderView = () => {
const { capabilities } = this.state;
return this.wrapInViewContainer(
'sql-data-loader',
<SqlDataLoaderView goToQuery={this.goToQuery} goToIngestion={this.goToIngestionWithTaskId} />,
<SqlDataLoaderView
capabilities={capabilities}
goToQuery={this.goToQuery}
goToIngestion={this.goToIngestionWithTaskId}
/>,
);
};

6
web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap
View File

@ -90,7 +90,7 @@ exports[`RetentionDialog matches snapshot 1`] = `
<span
class="bp4-button-text"
>
loadByPeriod(P1000Y+future)
loadByPeriod(P1000Y+future, 2x)
</span>
<span
aria-hidden="true"
@ -325,7 +325,7 @@ exports[`RetentionDialog matches snapshot 1`] = `
autocomplete="off"
class="bp4-input"
max="256"
min="1"
min="0"
type="text"
value="2"
/>
@ -562,7 +562,7 @@ exports[`RetentionDialog matches snapshot 1`] = `
<span
class="bp4-button-text"
>
loadForever
loadForever(2x)
</span>
</button>
</div>

4
web-console/src/druid-models/execution/execution.ts
View File

@ -106,6 +106,7 @@ function getUsageInfoFromStatusPayload(status: any): UsageInfo | undefined {
}
export interface CapacityInfo {
availableTaskSlots: number;
usedTaskSlots: number;
totalTaskSlots: number;
}
@ -125,8 +126,7 @@ function formatPendingMessage(
return baseMessage;
}
const { usedTaskSlots, totalTaskSlots } = capacityInfo;
const availableTaskSlots = totalTaskSlots - usedTaskSlots;
const { availableTaskSlots, usedTaskSlots, totalTaskSlots } = capacityInfo;
// If there are enough slots free: "Launched 2/4 tasks." (It will resolve very soon, no need to make it complicated.)
if (pendingTasks <= availableTaskSlots) {

5
web-console/src/druid-models/query-context/query-context.tsx
View File

@ -116,9 +116,8 @@ export function changeTimezone(context: QueryContext, timezone: string | undefin
// maxNumTasks
export function getMaxNumTasks(context: QueryContext): number {
const { maxNumTasks } = context;
return Math.max(typeof maxNumTasks === 'number' ? maxNumTasks : 0, 2);
export function getMaxNumTasks(context: QueryContext): number | undefined {
return context.maxNumTasks;
}
export function changeMaxNumTasks(

7
web-console/src/druid-models/workbench-query/workbench-query.ts
View File

@ -485,6 +485,13 @@ export class WorkbenchQuery {
return ret.changeQueryString(newQueryString);
}
public setMaxNumTasksIfUnset(maxNumTasks: number | undefined): WorkbenchQuery {
const { queryContext } = this;
if (typeof queryContext.maxNumTasks === 'number' || !maxNumTasks) return this;
return this.changeQueryContext({ ...queryContext, maxNumTasks: Math.max(maxNumTasks, 2) });
}
public getApiQuery(makeQueryId: () => string = uuidv4): {
engine: DruidEngine;
query: Record<string, any>;

48
web-console/src/utils/capabilities.ts → web-console/src/helpers/capabilities.ts
View File

@ -18,7 +18,7 @@
import { Api } from '../singletons';
import { localStorageGetJson, LocalStorageKeys } from './local-storage-keys';
import { maybeGetClusterCapacity } from './index';
export type CapabilitiesMode = 'full' | 'no-sql' | 'no-proxy';
@ -33,13 +33,12 @@ export type CapabilitiesModeExtended =
export type QueryType = 'none' | 'nativeOnly' | 'nativeAndSql';
export interface CapabilitiesOptions {
export interface CapabilitiesValue {
queryType: QueryType;
multiStageQuery: boolean;
coordinator: boolean;
overlord: boolean;
warnings?: string[];
clusterCapacity?: number;
}
export class Capabilities {
@ -55,6 +54,7 @@ export class Capabilities {
private readonly multiStageQuery: boolean;
private readonly coordinator: boolean;
private readonly overlord: boolean;
private readonly clusterCapacity?: number;
static async detectQueryType(): Promise<QueryType | undefined> {
// Check SQL endpoint
@ -137,9 +137,6 @@ export class Capabilities {
}
static async detectCapabilities(): Promise<Capabilities | undefined> {
const capabilitiesOverride = localStorageGetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE);
if (capabilitiesOverride) return new Capabilities(capabilitiesOverride);
const queryType = await Capabilities.detectQueryType();
if (typeof queryType === 'undefined') return;
@ -164,11 +161,34 @@ export class Capabilities {
});
}
constructor(options: CapabilitiesOptions) {
this.queryType = options.queryType;
this.multiStageQuery = options.multiStageQuery;
this.coordinator = options.coordinator;
this.overlord = options.overlord;
static async detectCapacity(capabilities: Capabilities): Promise<Capabilities> {
if (!capabilities.hasOverlordAccess()) return capabilities;
const capacity = await maybeGetClusterCapacity();
if (!capacity) return capabilities;
return new Capabilities({
...capabilities.valueOf(),
clusterCapacity: capacity.totalTaskSlots,
});
}
constructor(value: CapabilitiesValue) {
this.queryType = value.queryType;
this.multiStageQuery = value.multiStageQuery;
this.coordinator = value.coordinator;
this.overlord = value.overlord;
this.clusterCapacity = value.clusterCapacity;
}
public valueOf(): CapabilitiesValue {
return {
queryType: this.queryType,
multiStageQuery: this.multiStageQuery,
coordinator: this.coordinator,
overlord: this.overlord,
clusterCapacity: this.clusterCapacity,
};
}
public getMode(): CapabilitiesMode {
@ -240,6 +260,10 @@ export class Capabilities {
public hasSqlOrOverlordAccess(): boolean {
return this.hasSql() || this.hasOverlordAccess();
}
public getClusterCapacity(): number | undefined {
return this.clusterCapacity;
}
}
Capabilities.FULL = new Capabilities({
queryType: 'nativeAndSql',

6
web-console/src/helpers/capacity.ts
View File

@ -35,7 +35,11 @@ export async function getClusterCapacity(): Promise<CapacityInfo> {
Number(workerInfo.worker.capacity),
);
return { usedTaskSlots, totalTaskSlots };
return {
availableTaskSlots: totalTaskSlots - usedTaskSlots,
usedTaskSlots,
totalTaskSlots,
};
}
export async function maybeGetClusterCapacity(): Promise<CapacityInfo | undefined> {

1
web-console/src/helpers/index.ts
View File

@ -16,6 +16,7 @@
* limitations under the License.
*/
export * from './capabilities';
export * from './capacity';
export * from './execution/general';
export * from './execution/sql-task-execution';

1
web-console/src/hooks/index.ts
View File

@ -16,6 +16,7 @@
* limitations under the License.
*/
export * from './use-clock';
export * from './use-constant';
export * from './use-global-event-listener';
export * from './use-interval';

41
web-console/src/hooks/use-clock.ts Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { useEffect, useState } from 'react';
function getNowToSecond(): Date {
const now = new Date();
now.setMilliseconds(0);
return now;
}
export function useClock() {
const [now, setNow] = useState<Date>(getNowToSecond);
useEffect(() => {
const checkInterval = setInterval(() => {
setNow(getNowToSecond());
}, 1000);
return () => {
clearInterval(checkInterval);
};
}, []);
return now;
}

1
web-console/src/utils/index.tsx
View File

@ -16,7 +16,6 @@
* limitations under the License.
*/
export * from './capabilities';
export * from './column-metadata';
export * from './date';
export * from './download';

32
web-console/src/utils/load-rule.ts
View File

@ -53,11 +53,14 @@ export class RuleUtil {
];
static ruleToString(rule: Rule): string {
return [
rule.type,
rule.period ? `(${rule.period}${rule.includeFuture ? `+future` : ''})` : '',
rule.interval ? `(${rule.interval})` : '',
].join('');
const params: string[] = [];
if (RuleUtil.hasPeriod(rule))
params.push(`${rule.period}${rule.includeFuture ? '+future' : ''}`);
if (RuleUtil.hasInterval(rule)) params.push(rule.interval || '?');
if (RuleUtil.hasTieredReplicants(rule)) params.push(`${RuleUtil.totalReplicas(rule)}x`);
return `${rule.type}(${params.join(', ')})`;
}
static changeRuleType(rule: Rule, type: RuleType): Rule {
@ -121,4 +124,23 @@ export class RuleUtil {
const newTieredReplicants = deepSet(rule.tieredReplicants || {}, tier, replication);
return deepSet(rule, 'tieredReplicants', newTieredReplicants);
}
static totalReplicas(rule: Rule): number {
const tieredReplicants = rule.tieredReplicants || {};
let total = 0;
for (const k in tieredReplicants) {
total += tieredReplicants[k];
}
return total;
}
static isColdRule(rule: Rule): boolean {
return RuleUtil.hasTieredReplicants(rule) && RuleUtil.totalReplicas(rule) === 0;
}
static hasColdRule(rules: Rule[] | undefined, defaultRules: Rule[] | undefined): boolean {
return (
(rules || []).some(RuleUtil.isColdRule) || (defaultRules || []).some(RuleUtil.isColdRule)
);
}
}

2
web-console/src/views/datasources-view/datasources-view.spec.tsx
View File

@ -19,7 +19,7 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { DatasourcesView } from './datasources-view';

55
web-console/src/views/datasources-view/datasources-view.tsx
View File

@ -51,11 +51,10 @@ import {
QueryWithContext,
zeroCompactionStatus,
} from '../../druid-models';
import { Capabilities, CapabilitiesMode } from '../../helpers';
import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table';
import { Api, AppToaster } from '../../singletons';
import {
Capabilities,
CapabilitiesMode,
compact,
countBy,
deepGet,
@ -289,37 +288,37 @@ export class DatasourcesView extends React.PureComponent<
[
visibleColumns.shown('Datasource name') && `datasource`,
(visibleColumns.shown('Availability') || visibleColumns.shown('Segment granularity')) &&
`COUNT(*) FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS num_segments`,
`COUNT(*) FILTER (WHERE is_active = 1) AS num_segments`,
(visibleColumns.shown('Availability') || visibleColumns.shown('Availability detail')) && [
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load`,
`COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop`,
`COUNT(*) FILTER (WHERE is_available = 1 AND is_active = 0) AS num_segments_to_drop`,
],
visibleColumns.shown('Total data size') &&
`SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size`,
`SUM("size") FILTER (WHERE is_active = 1) AS total_data_size`,
visibleColumns.shown('Segment rows') && [
`MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows`,
`AVG("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_rows`,
`MAX("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_rows`,
`MIN("num_rows") FILTER (WHERE is_available = 1 AND is_realtime = 0) AS min_segment_rows`,
`AVG("num_rows") FILTER (WHERE is_available = 1 AND is_realtime = 0) AS avg_segment_rows`,
`MAX("num_rows") FILTER (WHERE is_available = 1 AND is_realtime = 0) AS max_segment_rows`,
],
visibleColumns.shown('Segment size') && [
`MIN("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_size`,
`AVG("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_size`,
`MAX("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_size`,
`MIN("size") FILTER (WHERE is_active = 1 AND is_realtime = 0) AS min_segment_size`,
`AVG("size") FILTER (WHERE is_active = 1 AND is_realtime = 0) AS avg_segment_size`,
`MAX("size") FILTER (WHERE is_active = 1 AND is_realtime = 0) AS max_segment_size`,
],
visibleColumns.shown('Segment granularity') && [
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments`,
`COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" = '-146136543-09-08T08:23:32.096Z' AND "end" = '146140482-04-24T15:36:27.903Z') AS all_granularity_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments`,
`COUNT(*) FILTER (WHERE is_active = 1 AND "start" = '-146136543-09-08T08:23:32.096Z' AND "end" = '146140482-04-24T15:36:27.903Z') AS all_granularity_segments`,
],
visibleColumns.shown('Total rows') &&
`SUM("num_rows") FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS total_rows`,
`SUM("num_rows") FILTER (WHERE is_active = 1) AS total_rows`,
visibleColumns.shown('Avg. row size') &&
`CASE WHEN SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) <> 0 THEN (SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) / SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0)) ELSE 0 END AS avg_row_size`,
`CASE WHEN SUM("num_rows") FILTER (WHERE is_available = 1) <> 0 THEN (SUM("size") FILTER (WHERE is_available = 1) / SUM("num_rows") FILTER (WHERE is_available = 1)) ELSE 0 END AS avg_row_size`,
visibleColumns.shown('Replicated size') &&
`SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size`,
`SUM("size" * "num_replicas") FILTER (WHERE is_active = 1) AS replicated_size`,
].flat(),
);
@ -1080,7 +1079,7 @@ ORDER BY 1`;
accessor: 'num_segments',
className: 'padded',
Cell: ({ value: num_segments, original }) => {
const { datasource, unused, num_segments_to_load } = original as Datasource;
const { datasource, unused, num_segments_to_load, rules } = original as Datasource;
if (unused) {
return (
<span>
@ -1090,6 +1089,7 @@ ORDER BY 1`;
);
}
const hasCold = RuleUtil.hasColdRule(rules, defaultRules);
const segmentsEl = (
<a onClick={() => goToSegments(datasource)}>
{pluralIfNeeded(num_segments, 'segment')}
@ -1104,13 +1104,17 @@ ORDER BY 1`;
Empty
</span>
);
} else if (num_segments_to_load === 0) {
} else if (num_segments_to_load === 0 || hasCold) {
const numAvailableSegments = num_segments - num_segments_to_load;
const percentHot = (
Math.floor((numAvailableSegments / num_segments) * 1000) / 10
).toFixed(1);
return (
<span>
<span style={{ color: DatasourcesView.FULLY_AVAILABLE_COLOR }}>
&#x25cf;&nbsp;
</span>
Fully available ({segmentsEl})
Fully available{hasCold ? `, ${percentHot}% hot` : ''} ({segmentsEl})
</span>
);
} else {
@ -1142,7 +1146,10 @@ ORDER BY 1`;
width: 180,
className: 'padded',
Cell: ({ original }) => {
const { num_segments_to_load, num_segments_to_drop } = original as Datasource;
const { num_segments_to_load, num_segments_to_drop, rules } = original as Datasource;
if (RuleUtil.hasColdRule(rules, defaultRules)) {
return pluralIfNeeded(num_segments_to_load, 'cold segment');
}
return formatLoadDrop(num_segments_to_load, num_segments_to_drop);
},
},

12
web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
View File

@ -8,6 +8,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
<Memo(DatasourcesCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": false,
"overlord": false,
@ -18,6 +19,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
<Memo(SegmentsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": false,
"overlord": false,
@ -28,6 +30,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
<Memo(ServicesCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": false,
"overlord": false,
@ -38,6 +41,7 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
<Memo(LookupsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": false,
"overlord": false,
@ -56,6 +60,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(DatasourcesCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -66,6 +71,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(SegmentsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -76,6 +82,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(SupervisorsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -86,6 +93,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(TasksCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -96,6 +104,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(ServicesCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -106,6 +115,7 @@ exports[`HomeView matches snapshot (full) 1`] = `
<Memo(LookupsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": true,
"multiStageQuery": true,
"overlord": true,
@ -124,6 +134,7 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
<Memo(SupervisorsCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": false,
"multiStageQuery": false,
"overlord": true,
@ -134,6 +145,7 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
<Memo(TasksCard)
capabilities={
Capabilities {
"clusterCapacity": undefined,
"coordinator": false,
"multiStageQuery": false,
"overlord": true,

2
web-console/src/views/home-view/datasources-card/datasources-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { DatasourcesCard } from './datasources-card';

5
web-console/src/views/home-view/datasources-card/datasources-card.tsx
View File

@ -19,9 +19,10 @@
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { pluralIfNeeded, queryDruidSql } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
export interface DatasourcesCardProps {
@ -31,7 +32,7 @@ export interface DatasourcesCardProps {
export const DatasourcesCard = React.memo(function DatasourcesCard(props: DatasourcesCardProps) {
const [datasourceCountState] = useQueryManager<Capabilities, number>({
processQuery: async capabilities => {
let datasources: string[];
let datasources: any[];
if (capabilities.hasSql()) {
datasources = await queryDruidSql({
query: `SELECT datasource FROM sys.segments GROUP BY 1`,

2
web-console/src/views/home-view/home-view-card/home-view-card.scss
View File

@ -21,7 +21,7 @@
.home-view-card {
.#{$bp-ns}-card {
height: 170px;
height: 200px;
}
&:hover {

2
web-console/src/views/home-view/home-view.spec.tsx
View File

@ -19,7 +19,7 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { HomeView } from './home-view';

2
web-console/src/views/home-view/home-view.tsx
View File

@ -18,7 +18,7 @@
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { DatasourcesCard } from './datasources-card/datasources-card';
import { LookupsCard } from './lookups-card/lookups-card';

2
web-console/src/views/home-view/lookups-card/lookups-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { LookupsCard } from './lookups-card';

3
web-console/src/views/home-view/lookups-card/lookups-card.tsx
View File

@ -20,9 +20,10 @@ import { IconNames } from '@blueprintjs/icons';
import { sum } from 'd3-array';
import React from 'react';
import { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, isLookupsUninitialized, pluralIfNeeded } from '../../../utils';
import { isLookupsUninitialized, pluralIfNeeded } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
export interface LookupsCardProps {

2
web-console/src/views/home-view/segments-card/segments-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { SegmentsCard } from './segments-card';

3
web-console/src/views/home-view/segments-card/segments-card.tsx
View File

@ -20,9 +20,10 @@ import { IconNames } from '@blueprintjs/icons';
import { sum } from 'd3-array';
import React from 'react';
import { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, deepGet, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { deepGet, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
export interface SegmentCounts {

2
web-console/src/views/home-view/services-card/services-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { ServicesCard } from './services-card';

3
web-console/src/views/home-view/services-card/services-card.tsx
View File

@ -20,9 +20,10 @@ import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import { PluralPairIfNeeded } from '../../../components';
import { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, lookupBy, queryDruidSql } from '../../../utils';
import { lookupBy, queryDruidSql } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
export interface ServiceCounts {

2
web-console/src/views/home-view/supervisors-card/supervisors-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { SupervisorsCard } from './supervisors-card';

3
web-console/src/views/home-view/supervisors-card/supervisors-card.tsx
View File

@ -19,9 +19,10 @@
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { pluralIfNeeded, queryDruidSql } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
export interface SupervisorCounts {

2
web-console/src/views/home-view/tasks-card/tasks-card.spec.tsx
View File

@ -19,7 +19,7 @@
import { render } from '@testing-library/react';
import React from 'react';
import { Capabilities } from '../../../utils';
import { Capabilities } from '../../../helpers';
import { TasksCard } from './tasks-card';

112
web-console/src/views/home-view/tasks-card/tasks-card.tsx
View File

@ -20,9 +20,11 @@ import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import { PluralPairIfNeeded } from '../../../components';
import { CapacityInfo } from '../../../druid-models';
import { Capabilities, getClusterCapacity } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
import { Capabilities, lookupBy, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { lookupBy, pluralIfNeeded, queryDruidSql } from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
function getTaskStatus(d: any) {
@ -30,55 +32,60 @@ function getTaskStatus(d: any) {
}
export interface TaskCounts {
SUCCESS?: number;
FAILED?: number;
RUNNING?: number;
PENDING?: number;
WAITING?: number;
success?: number;
failed?: number;
running?: number;
pending?: number;
waiting?: number;
}
async function getTaskCounts(capabilities: Capabilities): Promise<TaskCounts> {
if (capabilities.hasSql()) {
const taskCountsFromQuery = await queryDruidSql<{ status: string; count: number }>({
query: `SELECT
CASE WHEN "status" = 'RUNNING' THEN "runner_status" ELSE "status" END AS "status",
COUNT(*) AS "count"
FROM sys.tasks
GROUP BY 1`,
});
return lookupBy(
taskCountsFromQuery,
x => x.status.toLowerCase(),
x => x.count,
);
} else if (capabilities.hasOverlordAccess()) {
const tasks: any[] = (await Api.instance.get('/druid/indexer/v1/tasks')).data;
return {
success: tasks.filter(d => getTaskStatus(d) === 'SUCCESS').length,
failed: tasks.filter(d => getTaskStatus(d) === 'FAILED').length,
running: tasks.filter(d => getTaskStatus(d) === 'RUNNING').length,
pending: tasks.filter(d => getTaskStatus(d) === 'PENDING').length,
waiting: tasks.filter(d => getTaskStatus(d) === 'WAITING').length,
};
} else {
throw new Error(`must have SQL or overlord access`);
}
}
export interface TaskCountsAndCapacity extends TaskCounts, Partial<CapacityInfo> {}
export interface TasksCardProps {
capabilities: Capabilities;
}
export const TasksCard = React.memo(function TasksCard(props: TasksCardProps) {
const [taskCountState] = useQueryManager<Capabilities, TaskCounts>({
const [cardState] = useQueryManager<Capabilities, TaskCountsAndCapacity>({
processQuery: async capabilities => {
if (capabilities.hasSql()) {
const taskCountsFromQuery: { status: string; count: number }[] = await queryDruidSql({
query: `SELECT
CASE WHEN "status" = 'RUNNING' THEN "runner_status" ELSE "status" END AS "status",
COUNT (*) AS "count"
FROM sys.tasks
GROUP BY 1`,
});
return lookupBy(
taskCountsFromQuery,
x => x.status,
x => x.count,
);
} else if (capabilities.hasOverlordAccess()) {
const tasks: any[] = (await Api.instance.get('/druid/indexer/v1/tasks')).data;
return {
SUCCESS: tasks.filter(d => getTaskStatus(d) === 'SUCCESS').length,
FAILED: tasks.filter(d => getTaskStatus(d) === 'FAILED').length,
RUNNING: tasks.filter(d => getTaskStatus(d) === 'RUNNING').length,
PENDING: tasks.filter(d => getTaskStatus(d) === 'PENDING').length,
WAITING: tasks.filter(d => getTaskStatus(d) === 'WAITING').length,
};
} else {
throw new Error(`must have SQL or overlord access`);
}
const taskCounts = getTaskCounts(capabilities);
if (!capabilities.hasOverlordAccess()) return taskCounts;
const capacity = await getClusterCapacity();
return { ...taskCounts, ...capacity };
},
initQuery: props.capabilities,
});
const taskCounts = taskCountState.data;
const successTaskCount = taskCounts ? taskCounts.SUCCESS : 0;
const failedTaskCount = taskCounts ? taskCounts.FAILED : 0;
const runningTaskCount = taskCounts ? taskCounts.RUNNING : 0;
const pendingTaskCount = taskCounts ? taskCounts.PENDING : 0;
const waitingTaskCount = taskCounts ? taskCounts.WAITING : 0;
const { success, failed, running, pending, waiting, totalTaskSlots } = cardState.data || {};
return (
<HomeViewCard
@ -86,27 +93,26 @@ GROUP BY 1`,
href="#ingestion"
icon={IconNames.GANTT_CHART}
title="Tasks"
loading={taskCountState.loading}
error={taskCountState.error}
loading={cardState.loading}
error={cardState.error}
>
{Boolean(totalTaskSlots) && <p>{pluralIfNeeded(totalTaskSlots || 0, 'task slot')}</p>}
<PluralPairIfNeeded
firstCount={runningTaskCount}
firstCount={running}
firstSingular="running task"
secondCount={pendingTaskCount}
secondCount={pending}
secondSingular="pending task"
/>
{Boolean(successTaskCount) && (
<p>{pluralIfNeeded(successTaskCount || 0, 'successful task')}</p>
)}
{Boolean(waitingTaskCount) && <p>{pluralIfNeeded(waitingTaskCount || 0, 'waiting task')}</p>}
{Boolean(failedTaskCount) && <p>{pluralIfNeeded(failedTaskCount || 0, 'failed task')}</p>}
{Boolean(success) && <p>{pluralIfNeeded(success || 0, 'successful task')}</p>}
{Boolean(waiting) && <p>{pluralIfNeeded(waiting || 0, 'waiting task')}</p>}
{Boolean(failed) && <p>{pluralIfNeeded(failed || 0, 'failed task')}</p>}
{!(
Boolean(runningTaskCount) ||
Boolean(pendingTaskCount) ||
Boolean(successTaskCount) ||
Boolean(waitingTaskCount) ||
Boolean(failedTaskCount)
) && <p>There are no tasks</p>}
Boolean(running) ||
Boolean(pending) ||
Boolean(success) ||
Boolean(waiting) ||
Boolean(failed)
) && <p>No tasks</p>}
</HomeViewCard>
);
});

2
web-console/src/views/ingestion-view/ingestion-view.spec.tsx
View File

@ -19,7 +19,7 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { IngestionView } from './ingestion-view';

2
web-console/src/views/ingestion-view/ingestion-view.tsx
View File

@ -41,6 +41,7 @@ import {
TaskTableActionDialog,
} from '../../dialogs';
import { QueryWithContext } from '../../druid-models';
import { Capabilities } from '../../helpers';
import {
SMALL_TABLE_PAGE_SIZE,
SMALL_TABLE_PAGE_SIZE_OPTIONS,
@ -48,7 +49,6 @@ import {
} from '../../react-table';
import { Api, AppToaster } from '../../singletons';
import {
Capabilities,
deepGet,
formatDuration,
getDruidErrorMessage,

28
web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
View File

@ -64,9 +64,10 @@ exports[`SegmentsView matches snapshot 1`] = `
"Num rows",
"Avg. row size",
"Replicas",
"Is published",
"Is realtime",
"Is available",
"Is active",
"Is realtime",
"Is published",
"Is overshadowed",
"Actions",
]
@ -76,6 +77,8 @@ exports[`SegmentsView matches snapshot 1`] = `
tableColumnsHidden={
Array [
"Time span",
"Is published",
"Is overshadowed",
]
}
/>
@ -277,10 +280,19 @@ exports[`SegmentsView matches snapshot 1`] = `
},
Object {
"Filter": [Function],
"Header": "Is published",
"Header": "Is available",
"accessor": [Function],
"className": "padded",
"id": "is_published",
"id": "is_available",
"show": true,
"width": 100,
},
Object {
"Filter": [Function],
"Header": "Is active",
"accessor": [Function],
"className": "padded",
"id": "is_active",
"show": true,
"width": 100,
},
@ -295,11 +307,11 @@ exports[`SegmentsView matches snapshot 1`] = `
},
Object {
"Filter": [Function],
"Header": "Is available",
"Header": "Is published",
"accessor": [Function],
"className": "padded",
"id": "is_available",
"show": true,
"id": "is_published",
"show": false,
"width": 100,
},
Object {
@ -308,7 +320,7 @@ exports[`SegmentsView matches snapshot 1`] = `
"accessor": [Function],
"className": "padded",
"id": "is_overshadowed",
"show": true,
"show": false,
"width": 100,
},
Object {

2
web-console/src/views/segments-view/segments-view.spec.tsx
View File

@ -19,7 +19,7 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities } from '../../utils';
import { Capabilities } from '../../helpers';
import { SegmentsView } from '../segments-view/segments-view';
describe('SegmentsView', () => {

49
web-console/src/views/segments-view/segments-view.tsx
View File

@ -42,6 +42,7 @@ import { AsyncActionDialog } from '../../dialogs';
import { SegmentTableActionDialog } from '../../dialogs/segments-table-action-dialog/segment-table-action-dialog';
import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
import { QueryWithContext } from '../../druid-models';
import { Capabilities, CapabilitiesMode } from '../../helpers';
import {
booleanCustomTableFilter,
BooleanFilterInput,
@ -52,8 +53,6 @@ import {
} from '../../react-table';
import { Api } from '../../singletons';
import {
Capabilities,
CapabilitiesMode,
compact,
deepGet,
filterMap,
@ -88,9 +87,10 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Num rows',
'Avg. row size',
'Replicas',
'Is published',
'Is realtime',
'Is available',
'Is active',
'Is realtime',
'Is published',
'Is overshadowed',
ACTION_COLUMN_LABEL,
],
@ -117,9 +117,10 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'Num rows',
'Avg. row size',
'Replicas',
'Is published',
'Is realtime',
'Is available',
'Is active',
'Is realtime',
'Is published',
'Is overshadowed',
],
};
@ -168,8 +169,9 @@ interface SegmentQueryResultRow {
avg_row_size: NumberLike;
num_replicas: number;
is_available: number;
is_published: number;
is_active: number;
is_realtime: number;
is_published: number;
is_overshadowed: number;
}
@ -212,9 +214,10 @@ END AS "time_span"`,
visibleColumns.shown('Avg. row size') &&
`CASE WHEN "num_rows" <> 0 THEN ("size" / "num_rows") ELSE 0 END AS "avg_row_size"`,
visibleColumns.shown('Replicas') && `"num_replicas"`,
visibleColumns.shown('Is published') && `"is_published"`,
visibleColumns.shown('Is available') && `"is_available"`,
visibleColumns.shown('Is active') && `"is_active"`,
visibleColumns.shown('Is realtime') && `"is_realtime"`,
visibleColumns.shown('Is published') && `"is_published"`,
visibleColumns.shown('Is overshadowed') && `"is_overshadowed"`,
]);
@ -262,7 +265,7 @@ END AS "time_span"`,
segmentFilter,
visibleColumns: new LocalStorageBackedVisibility(
LocalStorageKeys.SEGMENT_TABLE_COLUMN_SELECTION,
['Time span'],
['Time span', 'Is published', 'Is overshadowed'],
),
groupByInterval: false,
showSegmentTimeline: false,
@ -416,8 +419,9 @@ END AS "time_span"`,
avg_row_size: -1,
num_replicas: -1,
is_available: -1,
is_published: -1,
is_active: -1,
is_realtime: -1,
is_published: -1,
is_overshadowed: -1,
};
});
@ -813,10 +817,19 @@ END AS "time_span"`,
className: 'padded',
},
{
Header: 'Is published',
show: hasSql && visibleColumns.shown('Is published'),
id: 'is_published',
accessor: row => String(Boolean(row.is_published)),
Header: 'Is available',
show: hasSql && visibleColumns.shown('Is available'),
id: 'is_available',
accessor: row => String(Boolean(row.is_available)),
Filter: BooleanFilterInput,
className: 'padded',
width: 100,
},
{
Header: 'Is active',
show: hasSql && visibleColumns.shown('Is active'),
id: 'is_active',
accessor: row => String(Boolean(row.is_active)),
Filter: BooleanFilterInput,
className: 'padded',
width: 100,
@ -831,10 +844,10 @@ END AS "time_span"`,
width: 100,
},
{
Header: 'Is available',
show: hasSql && visibleColumns.shown('Is available'),
id: 'is_available',
accessor: row => String(Boolean(row.is_available)),
Header: 'Is published',
show: hasSql && visibleColumns.shown('Is published'),
id: 'is_published',
accessor: row => String(Boolean(row.is_published)),
Filter: BooleanFilterInput,
className: 'padded',
width: 100,

3
web-console/src/views/services-view/services-view.spec.tsx
View File

@ -19,7 +19,8 @@
import { shallow } from 'enzyme';
import React from 'react';
import { Capabilities, QueryState } from '../../utils';
import { Capabilities } from '../../helpers';
import { QueryState } from '../../utils';
import { ServicesView } from './services-view';

3
web-console/src/views/services-view/services-view.tsx
View File

@ -35,11 +35,10 @@ import {
} from '../../components';
import { AsyncActionDialog } from '../../dialogs';
import { QueryWithContext } from '../../druid-models';
import { Capabilities, CapabilitiesMode } from '../../helpers';
import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table';
import { Api, AppToaster } from '../../singletons';
import {
Capabilities,
CapabilitiesMode,
deepGet,
filterMap,
formatBytes,

25
web-console/src/views/sql-data-loader-view/ingestion-progress-dialog/ingestion-progress-dialog.tsx
View File

@ -95,15 +95,22 @@ export const IngestionProgressDialog = React.memo(function IngestionProgressDial
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
{insertResultState.isLoading() && (
<Button
icon={IconNames.GANTT_CHART}
text="Go to Ingestion view"
rightIcon={IconNames.ARROW_TOP_RIGHT}
onClick={() => {
if (!insertResultState.intermediate) return;
goToIngestion(insertResultState.intermediate.id);
}}
/>
<>
<Button
icon={IconNames.RESET}
text="Ingest in background (reset data loader)"
onClick={onReset}
/>
<Button
icon={IconNames.GANTT_CHART}
text="Go to Ingestion view"
rightIcon={IconNames.ARROW_TOP_RIGHT}
onClick={() => {
if (!insertResultState.intermediate) return;
goToIngestion(insertResultState.intermediate.id);
}}
/>
</>
)}
{insertResultState.isError() && <Button text="Close" onClick={onClose} />}
{insertResultState.data && (

73
web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
View File

@ -26,12 +26,14 @@ import {
ExternalConfig,
externalConfigToIngestQueryPattern,
ingestQueryPatternToQuery,
QueryContext,
QueryWithContext,
} from '../../druid-models';
import { submitTaskQuery } from '../../helpers';
import { Capabilities, maybeGetClusterCapacity, submitTaskQuery } from '../../helpers';
import { useLocalStorageState } from '../../hooks';
import { AppToaster } from '../../singletons';
import { deepDelete, LocalStorageKeys } from '../../utils';
import { CapacityAlert } from '../workbench-view/capacity-alert/capacity-alert';
import { InputFormatStep } from '../workbench-view/input-format-step/input-format-step';
import { InputSourceStep } from '../workbench-view/input-source-step/input-source-step';
import { MaxTasksButton } from '../workbench-view/max-tasks-button/max-tasks-button';
@ -47,6 +49,7 @@ interface LoaderContent extends QueryWithContext {
}
export interface SqlDataLoaderViewProps {
capabilities: Capabilities;
goToQuery(queryWithContext: QueryWithContext): void;
goToIngestion(taskId: string): void;
}
@ -54,7 +57,8 @@ export interface SqlDataLoaderViewProps {
export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
props: SqlDataLoaderViewProps,
) {
const { goToQuery, goToIngestion } = props;
const { capabilities, goToQuery, goToIngestion } = props;
const [alertElement, setAlertElement] = useState<JSX.Element | undefined>();
const [externalConfigStep, setExternalConfigStep] = useState<Partial<ExternalConfig>>({});
const [content, setContent] = useLocalStorageState<LoaderContent | undefined>(
LocalStorageKeys.SQL_DATA_LOADER_CONTENT,
@ -75,6 +79,26 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
);
}
async function submitTask(query: string, context: QueryContext) {
if (!content) return;
try {
const execution = await submitTaskQuery({
query,
context,
});
const taskId = execution instanceof Execution ? execution.id : execution.state.id;
setContent({ ...content, id: taskId });
} catch (e) {
AppToaster.show({
message: `Error submitting task: ${e.message}`,
intent: Intent.DANGER,
});
}
}
return (
<div className="sql-data-loader-view">
{needVerify ? (
@ -105,33 +129,47 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
goToQuery={() => goToQuery(content)}
onBack={() => setContent(undefined)}
onDone={async () => {
const ingestDatasource = SqlQuery.parse(content.queryString)
.getIngestTable()
?.getName();
const { queryString, queryContext } = content;
const ingestDatasource = SqlQuery.parse(queryString).getIngestTable()?.getName();
if (!ingestDatasource) {
AppToaster.show({ message: `Must have an ingest datasource`, intent: Intent.DANGER });
return;
}
try {
const execution = await submitTaskQuery({
query: content.queryString,
context: content.queryContext,
});
const clusterCapacity = capabilities.getClusterCapacity();
let effectiveContext = queryContext || {};
if (
typeof effectiveContext.maxNumTasks === 'undefined' &&
typeof clusterCapacity === 'number'
) {
effectiveContext = { ...effectiveContext, maxNumTasks: clusterCapacity };
}
const taskId = execution instanceof Execution ? execution.id : execution.state.id;
const capacityInfo = await maybeGetClusterCapacity();
setContent({ ...content, id: taskId });
} catch (e) {
AppToaster.show({
message: `Error submitting task: ${e.message}`,
intent: Intent.DANGER,
});
const effectiveMaxNumTasks = effectiveContext.maxNumTasks ?? 2;
if (capacityInfo && capacityInfo.availableTaskSlots < effectiveMaxNumTasks) {
setAlertElement(
<CapacityAlert
maxNumTasks={effectiveMaxNumTasks}
capacityInfo={capacityInfo}
onRun={() => {
void submitTask(queryString, effectiveContext);
}}
onClose={() => {
setAlertElement(undefined);
}}
/>,
);
} else {
await submitTask(queryString, effectiveContext);
}
}}
extraCallout={
<MaxTasksButton
clusterCapacity={capabilities.getClusterCapacity()}
queryContext={content.queryContext || {}}
changeQueryContext={queryContext => setContent({ ...content, queryContext })}
minimal
@ -198,6 +236,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
onClose={() => setContent(deepDelete(content, 'id'))}
/>
)}
{alertElement}
</div>
);
});

6
web-console/src/views/workbench-view/capacity-alert/capacity-alert.tsx
View File

@ -70,9 +70,9 @@ export function CapacityAlert(props: CapacityAlertProps) {
>
<p>
The cluster does not currently have enough available task slots (current usage:{' '}
<Code>{`${formatInteger(usedTaskSlots)}/${formatInteger(totalTaskSlots)}`}</Code>) to run
this query which is set to use up to <Code>{formatInteger(maxNumTasks)}</Code> tasks. This
query might have to wait for task slots to free up before running.
<Code>{`${formatInteger(usedTaskSlots)} of ${formatInteger(totalTaskSlots)}`}</Code>) to
run this query which is set to use up to <Code>{formatInteger(maxNumTasks)}</Code> tasks.
This query might have to wait for task slots to free up before running.
</p>
<p>Are you sure you want to run it?</p>
</Alert>

16
web-console/src/views/workbench-view/execution-stages-pane/__snapshots__/execution-stages-pane.spec.tsx.snap
View File

@ -84,30 +84,30 @@ exports[`ExecutionStagesPane matches snapshot 1`] = `
Object {
"Cell": [Function],
"Header": <React.Fragment>
Data processed
Rows processed
<br />
<i>
rows   (size or files)
rows   (input files)
</i>
</React.Fragment>,
"accessor": [Function],
"className": "padded",
"id": "data_processed",
"width": 220,
"id": "rows_processed",
"width": 160,
},
Object {
"Cell": [Function],
"Header": <React.Fragment>
Data processing rate
Processing rate
<br />
<i>
rows/s   (data rate)
rows/s
</i>
</React.Fragment>,
"accessor": [Function],
"className": "padded",
"id": "data_processing_rate",
"width": 200,
"id": "processing_rate",
"width": 150,
},
Object {
"Header": "Phase",

5
web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.scss
View File

@ -71,11 +71,6 @@
text-align: right;
padding-right: 15px;
}
.sort-percent {
display: inline-block;
margin-left: 20px;
}
}
.execution-stage-detail-pane {

160
web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx
View File

@ -26,6 +26,7 @@ import ReactTable, { Column } from 'react-table';
import { BracedText, TableClickableCell } from '../../../components';
import {
ChannelCounterName,
ChannelFields,
ClusterBy,
CounterName,
Execution,
@ -40,7 +41,8 @@ import {
capitalizeFirst,
clamp,
deepGet,
formatBytes,
filterMap,
formatBytesCompact,
formatDuration,
formatDurationWithMs,
formatInteger,
@ -54,6 +56,7 @@ import './execution-stages-pane.scss';
const MAX_STAGE_ROWS = 20;
const MAX_DETAIL_ROWS = 20;
const NOT_SIZE_ON_DISK = '(does not represent size on disk)';
function formatBreakdown(breakdown: Record<string, number>): string {
return Object.keys(breakdown)
@ -63,8 +66,6 @@ function formatBreakdown(breakdown: Record<string, number>): string {
const formatRows = formatInteger;
const formatRowRate = formatInteger;
const formatSize = (bytes: number) => `(${formatBytes(bytes)})`;
const formatByteRate = (byteRate: number) => `(${formatBytes(byteRate)}/s)`;
const formatFrames = formatInteger;
const formatDurationDynamic = (n: NumberLike) =>
n < 1000 ? formatDurationWithMs(n) : formatDuration(n);
@ -120,9 +121,6 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
const rowRateValues = stages.stages.map(s =>
formatRowRate(stages.getRateFromStage(s, 'rows') || 0),
);
const byteRateValues = stages.stages.map(s =>
formatByteRate(stages.getRateFromStage(s, 'bytes') || 0),
);
const rowsValues = stages.stages.flatMap(stage => [
...stages.getInputCountersForStage(stage, 'rows').map(formatRows),
@ -130,14 +128,10 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
formatRows(stages.getTotalCounterForStage(stage, 'shuffle', 'rows')),
]);
const bytesAndFilesValues = stages.stages.flatMap(stage => {
const filesValues = filterMap(stages.stages, stage => {
const inputFileCount = stages.getTotalInputForStage(stage, 'totalFiles');
return [
...stages.getInputCountersForStage(stage, 'bytes').map(formatSize),
formatSize(stages.getTotalCounterForStage(stage, 'output', 'bytes')),
formatSize(stages.getTotalCounterForStage(stage, 'shuffle', 'bytes')),
inputFileCount ? formatFileOfTotalForBrace(inputFileCount, inputFileCount) : '',
];
if (!inputFileCount) return;
return formatFileOfTotalForBrace(inputFileCount, inputFileCount);
});
function detailedStats(stage: StageDefinition) {
@ -164,13 +158,10 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
bracesRows[counterName] = wideCounters.map(wideCounter =>
formatRows(wideCounter[counterName]!.rows),
);
bracesExtra[counterName] = wideCounters.map(wideCounter => {
bracesExtra[counterName] = filterMap(wideCounters, wideCounter => {
const totalFiles = wideCounter[counterName]!.totalFiles;
if (totalFiles) {
return formatFileOfTotalForBrace(totalFiles, totalFiles);
} else {
return formatSize(wideCounter[counterName]!.bytes);
}
if (!totalFiles) return;
return formatFileOfTotalForBrace(totalFiles, totalFiles);
});
}
@ -212,7 +203,7 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
) : (
stages.getStageCounterTitle(stage, counterName)
),
isInput ? <i>rows &nbsp; (size or files)</i> : <i>rows &nbsp; (size)</i>,
isInput ? <i>rows &nbsp; (input files)</i> : <i>rows</i>,
),
id: counterName,
accessor: d => d[counterName]!.rows,
@ -222,8 +213,16 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
const c = (original as SimpleWideCounter)[counterName]!;
return (
<>
<BracedText text={formatRows(value)} braces={bracesRows[counterName]} />
{c.totalFiles ? (
<BracedText
text={formatRows(value)}
braces={bracesRows[counterName]}
title={
c.bytes
? `Uncompressed size: ${formatBytesCompact(c.bytes)} ${NOT_SIZE_ON_DISK}`
: undefined
}
/>
{Boolean(c.totalFiles) && (
<>
{' '}
&nbsp;{' '}
@ -232,13 +231,7 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
braces={bracesExtra[counterName]}
/>
</>
) : c.bytes ? (
<>
{' '}
&nbsp;
<BracedText text={formatSize(c.bytes)} braces={bracesExtra[counterName]} />
</>
) : undefined}
)}
</>
);
},
@ -263,14 +256,10 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
);
const bracesRows: Record<ChannelCounterName, string[]> = {} as any;
const bracesBytes: Record<ChannelCounterName, string[]> = {} as any;
for (const counterName of counterNames) {
bracesRows[counterName] = wideCounters.map(wideCounter =>
formatRows(wideCounter[counterName]!.rows),
);
bracesBytes[counterName] = wideCounters.map(wideCounter =>
formatSize(wideCounter[counterName]!.bytes),
);
}
return (
@ -305,18 +294,17 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
className: 'padded',
width: 180,
Cell({ value, original }) {
const c = original[counterName];
const c: Record<ChannelFields, number> = original[counterName];
return (
<>
<BracedText text={formatRows(value)} braces={bracesRows[counterName]} />
{c.bytes ? (
<>
{' '}
&nbsp;
<BracedText text={formatSize(c.bytes)} braces={bracesBytes[counterName]} />
</>
) : undefined}
</>
<BracedText
text={formatRows(value)}
braces={bracesRows[counterName]}
title={
c.bytes
? `Uncompressed size: ${formatBytesCompact(c.bytes)} ${NOT_SIZE_ON_DISK}`
: undefined
}
/>
);
},
};
@ -329,11 +317,21 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
function dataProcessedInput(stage: StageDefinition, inputNumber: number) {
const inputCounter: CounterName = `input${inputNumber}`;
if (!stages.hasCounterForStage(stage, inputCounter)) return;
const inputSizeBytes = stages.getTotalCounterForStage(stage, inputCounter, 'bytes');
const inputFileCount = stages.getTotalCounterForStage(stage, inputCounter, 'totalFiles');
const bytes = stages.getTotalCounterForStage(stage, inputCounter, 'bytes');
return (
<div className="data-transfer" key={inputNumber}>
<div
className="data-transfer"
key={inputNumber}
title={
bytes
? `Input${inputNumber} uncompressed size: ${formatBytesCompact(
bytes,
)} ${NOT_SIZE_ON_DISK}`
: undefined
}
>
<BracedText
text={formatRows(stages.getTotalCounterForStage(stage, inputCounter, 'rows'))}
braces={rowsValues}
@ -347,14 +345,9 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
stages.getTotalCounterForStage(stage, inputCounter, 'files'),
inputFileCount,
)}
braces={bytesAndFilesValues}
braces={filesValues}
/>
</>
) : inputSizeBytes ? (
<>
{' '}
&nbsp; <BracedText text={formatSize(inputSizeBytes)} braces={bytesAndFilesValues} />
</>
) : undefined}
</div>
);
@ -383,21 +376,20 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
function dataProcessedOutput(stage: StageDefinition) {
if (!stages.hasCounterForStage(stage, 'output')) return;
const title = stages.getStageCounterTitle(stage, 'output');
return (
<div
className="data-transfer"
title={`${stages.getStageCounterTitle(stage, 'output')} frames: ${formatFrames(
title={`${title} frames: ${formatFrames(
stages.getTotalCounterForStage(stage, 'output', 'frames'),
)}`}
)}
${title} uncompressed size: ${formatBytesCompact(
stages.getTotalCounterForStage(stage, 'output', 'bytes'),
)} ${NOT_SIZE_ON_DISK}`}
>
<BracedText
text={formatRows(stages.getTotalCounterForStage(stage, 'output', 'rows'))}
braces={rowsValues}
/>{' '}
&nbsp;{' '}
<BracedText
text={formatSize(stages.getTotalCounterForStage(stage, 'output', 'bytes'))}
braces={bytesAndFilesValues}
/>
</div>
);
@ -410,26 +402,20 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
const shuffleRows = stages.getTotalCounterForStage(stage, 'shuffle', 'rows');
const sortProgress = stages.getSortProgressForStage(stage);
const title = stages.getStageCounterTitle(stage, 'shuffle');
return (
<div
className="data-transfer"
title={`${stages.getStageCounterTitle(stage, 'shuffle')} frames: ${formatFrames(
title={`${title} frames: ${formatFrames(
stages.getTotalCounterForStage(stage, 'shuffle', 'frames'),
)}`}
)}
${title} uncompressed size: ${formatBytesCompact(
stages.getTotalCounterForStage(stage, 'shuffle', 'bytes'),
)} ${NOT_SIZE_ON_DISK}`}
>
{shuffleRows ? (
<>
<BracedText text={formatRows(shuffleRows)} braces={rowsValues} /> &nbsp;{' '}
<BracedText
text={formatSize(stages.getTotalCounterForStage(stage, 'shuffle', 'bytes'))}
braces={bytesAndFilesValues}
/>
{0 < sortProgress && sortProgress < 1 && (
<div className="sort-percent">{`[${formatPercent(sortProgress)}]`}</div>
)}
</>
) : (
<BracedText text={`[${formatPercent(sortProgress)}]`} braces={rowsValues} />
<BracedText text={shuffleRows ? formatRows(shuffleRows) : ''} braces={rowsValues} /> &nbsp;{' '}
{0 < sortProgress && sortProgress < 1 && (
<> &nbsp;{` ${formatPercent(sortProgress)} sorted`}</>
)}
</div>
);
@ -535,11 +521,11 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
},
},
{
Header: twoLines('Data processed', <i>rows &nbsp; (size or files)</i>),
id: 'data_processed',
Header: twoLines('Rows processed', <i>rows &nbsp; (input files)</i>),
id: 'rows_processed',
accessor: () => null,
className: 'padded',
width: 220,
width: 160,
Cell({ original }) {
const stage = original as StageDefinition;
const { input, broadcast } = stage.definition;
@ -560,26 +546,22 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane(
},
},
{
Header: twoLines('Data processing rate', <i>rows/s &nbsp; (data rate)</i>),
id: 'data_processing_rate',
Header: twoLines('Processing rate', <i>rows/s</i>),
id: 'processing_rate',
accessor: s => stages.getRateFromStage(s, 'rows'),
className: 'padded',
width: 200,
width: 150,
Cell({ value, original }) {
const stage = original as StageDefinition;
if (typeof value !== 'number') return null;
const byteRate = stages.getRateFromStage(stage, 'bytes');
return (
<>
<BracedText text={formatRowRate(value)} braces={rowRateValues} />
{byteRate ? (
<>
{' '}
&nbsp; <BracedText text={formatByteRate(byteRate)} braces={byteRateValues} />
</>
) : undefined}
</>
<BracedText
text={formatRowRate(value)}
braces={rowRateValues}
title={byteRate ? `${formatBytesCompact(byteRate)}/s` : undefined}
/>
);
},
},

12
web-console/src/views/workbench-view/helper-query/helper-query.tsx
View File

@ -77,6 +77,7 @@ export interface HelperQueryProps {
onDelete(): void;
onDetails(id: string, initTab?: ExecutionDetailsTab): void;
queryEngines: DruidEngine[];
clusterCapacity: number | undefined;
goToIngestion(taskId: string): void;
}
@ -90,6 +91,7 @@ export const HelperQuery = React.memo(function HelperQuery(props: HelperQueryPro
onDelete,
onDetails,
queryEngines,
clusterCapacity,
goToIngestion,
} = props;
const [alertElement, setAlertElement] = useState<JSX.Element | undefined>();
@ -252,15 +254,14 @@ export const HelperQuery = React.memo(function HelperQuery(props: HelperQueryPro
return;
}
const effectiveQuery = preview ? query.makePreview() : query;
const effectiveQuery = preview
? query.makePreview()
: query.setMaxNumTasksIfUnset(clusterCapacity);
const capacityInfo = await maybeGetClusterCapacity();
const effectiveMaxNumTasks = effectiveQuery.queryContext.maxNumTasks ?? 2;
if (
capacityInfo &&
capacityInfo.totalTaskSlots - capacityInfo.usedTaskSlots < effectiveMaxNumTasks
) {
if (capacityInfo && capacityInfo.availableTaskSlots < effectiveMaxNumTasks) {
setAlertElement(
<CapacityAlert
maxNumTasks={effectiveMaxNumTasks}
@ -367,6 +368,7 @@ export const HelperQuery = React.memo(function HelperQuery(props: HelperQueryPro
loading={executionState.loading}
small
queryEngines={queryEngines}
clusterCapacity={clusterCapacity}
/>
{executionState.isLoading() && (
<ExecutionTimerPanel

87
web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap
View File

@ -9,12 +9,23 @@ exports[`MaxTasksButton matches snapshot 1`] = `
content={
<Blueprint4.Menu>
<Blueprint4.MenuDivider
title="Number of tasks to launch"
title="Maximum number of tasks to launch"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="tick"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="6 (full cluster capacity)"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + 1 worker)"
multiline={false}
onClick={[Function]}
@ -59,78 +70,6 @@ exports[`MaxTasksButton matches snapshot 1`] = `
shouldDismissPopover={true}
text="5"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 6 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="7"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 8 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="9"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 10 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="11"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 16 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="17"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 32 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="33"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
icon="blank"
label="(1 controller + max 64 workers)"
multiline={false}
onClick={[Function]}
popoverProps={Object {}}
selected={false}
shouldDismissPopover={true}
text="65"
/>
<Blueprint4.MenuItem
active={false}
disabled={false}
@ -198,7 +137,7 @@ exports[`MaxTasksButton matches snapshot 1`] = `
>
<Blueprint4.Button
rightIcon="caret-down"
text="Max tasks: 2"
text="Max tasks: 6 (full cluster capacity)"
/>
</Blueprint4.Popover2>
</Fragment>

4
web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.spec.tsx
View File

@ -23,7 +23,9 @@ import { MaxTasksButton } from './max-tasks-button';
describe('MaxTasksButton', () => {
it('matches snapshot', () => {
const comp = shallow(<MaxTasksButton queryContext={{}} changeQueryContext={() => {}} />);
const comp = shallow(
<MaxTasksButton clusterCapacity={6} queryContext={{}} changeQueryContext={() => {}} />,
);
expect(comp).toMatchSnapshot();
});

38
web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx
View File

@ -31,7 +31,7 @@ import {
} from '../../../druid-models';
import { formatInteger, tickIcon } from '../../../utils';
const MAX_NUM_TASK_OPTIONS = [2, 3, 4, 5, 7, 9, 11, 17, 33, 65];
const MAX_NUM_TASK_OPTIONS = [2, 3, 4, 5, 7, 9, 11, 17, 33, 65, 129];
const TASK_ASSIGNMENT_OPTIONS = ['max', 'auto'];
const TASK_ASSIGNMENT_DESCRIPTION: Record<string, string> = {
@ -40,17 +40,22 @@ const TASK_ASSIGNMENT_DESCRIPTION: Record<string, string> = {
};
export interface MaxTasksButtonProps extends Omit<ButtonProps, 'text' | 'rightIcon'> {
clusterCapacity: number | undefined;
queryContext: QueryContext;
changeQueryContext(queryContext: QueryContext): void;
}
export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps) {
const { queryContext, changeQueryContext, ...rest } = props;
const { clusterCapacity, queryContext, changeQueryContext, ...rest } = props;
const [customMaxNumTasksDialogOpen, setCustomMaxNumTasksDialogOpen] = useState(false);
const maxNumTasks = getMaxNumTasks(queryContext);
const taskAssigment = getTaskAssigment(queryContext);
const fullClusterCapacity = `${clusterCapacity} (full cluster capacity)`;
const shownMaxNumTaskOptions = clusterCapacity
? MAX_NUM_TASK_OPTIONS.filter(_ => _ <= clusterCapacity)
: MAX_NUM_TASK_OPTIONS;
return (
<>
<Popover2
@ -58,8 +63,15 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
position={Position.BOTTOM_LEFT}
content={
<Menu>
<MenuDivider title="Number of tasks to launch" />
{MAX_NUM_TASK_OPTIONS.map(m => (
<MenuDivider title="Maximum number of tasks to launch" />
{Boolean(clusterCapacity) && (
<MenuItem
icon={tickIcon(typeof maxNumTasks === 'undefined')}
text={fullClusterCapacity}
onClick={() => changeQueryContext(changeMaxNumTasks(queryContext, undefined))}
/>
)}
{shownMaxNumTaskOptions.map(m => (
<MenuItem
key={String(m)}
icon={tickIcon(m === maxNumTasks)}
@ -69,7 +81,9 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
/>
))}
<MenuItem
icon={tickIcon(!MAX_NUM_TASK_OPTIONS.includes(maxNumTasks))}
icon={tickIcon(
typeof maxNumTasks === 'number' && !shownMaxNumTaskOptions.includes(maxNumTasks),
)}
text="Custom"
onClick={() => setCustomMaxNumTasksDialogOpen(true)}
/>
@ -88,7 +102,17 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
</Menu>
}
>
<Button {...rest} text={`Max tasks: ${maxNumTasks}`} rightIcon={IconNames.CARET_DOWN} />
<Button
{...rest}
text={`Max tasks: ${
typeof maxNumTasks === 'undefined'
? clusterCapacity
? fullClusterCapacity
: 2
: maxNumTasks
}`}
rightIcon={IconNames.CARET_DOWN}
/>
</Popover2>
{customMaxNumTasksDialogOpen && (
<NumericInputDialog
@ -104,7 +128,7 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
}
minValue={2}
integer
initValue={maxNumTasks}
initValue={maxNumTasks || 2}
onSubmit={p => {
changeQueryContext(changeMaxNumTasks(queryContext, p));
}}

13
web-console/src/views/workbench-view/query-tab/query-tab.tsx
View File

@ -87,6 +87,7 @@ export interface QueryTabProps {
onDetails(id: string, initTab?: ExecutionDetailsTab): void;
queryEngines: DruidEngine[];
runMoreMenu: JSX.Element;
clusterCapacity: number | undefined;
goToIngestion(taskId: string): void;
}
@ -100,6 +101,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
onDetails,
queryEngines,
runMoreMenu,
clusterCapacity,
goToIngestion,
} = props;
const [alertElement, setAlertElement] = useState<JSX.Element | undefined>();
@ -281,15 +283,14 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
return;
}
const effectiveQuery = preview ? query.makePreview() : query;
const effectiveQuery = preview
? query.makePreview()
: query.setMaxNumTasksIfUnset(clusterCapacity);
const capacityInfo = await maybeGetClusterCapacity();
const effectiveMaxNumTasks = effectiveQuery.queryContext.maxNumTasks ?? 2;
if (
capacityInfo &&
capacityInfo.totalTaskSlots - capacityInfo.usedTaskSlots < effectiveMaxNumTasks
) {
if (capacityInfo && capacityInfo.availableTaskSlots < effectiveMaxNumTasks) {
setAlertElement(
<CapacityAlert
maxNumTasks={effectiveMaxNumTasks}
@ -344,6 +345,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
}}
onDetails={onDetails}
queryEngines={queryEngines}
clusterCapacity={clusterCapacity}
goToIngestion={goToIngestion}
/>
))}
@ -404,6 +406,7 @@ export const QueryTab = React.memo(function QueryTab(props: QueryTabProps) {
onRun={handleRun}
loading={executionState.loading}
queryEngines={queryEngines}
clusterCapacity={clusterCapacity}
moreMenu={runMoreMenu}
/>
{executionState.isLoading() && (

11
web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx
View File

@ -27,7 +27,7 @@ import React, { useState } from 'react';
import { Loader } from '../../../components';
import { Execution, WorkbenchQuery } from '../../../druid-models';
import { cancelTaskExecution, getTaskExecution } from '../../../helpers';
import { useInterval, useQueryManager } from '../../../hooks';
import { useClock, useInterval, useQueryManager } from '../../../hooks';
import { AppToaster } from '../../../singletons';
import { downloadQueryDetailArchive, formatDuration, queryDruidSql } from '../../../utils';
import { CancelQueryDialog } from '../cancel-query-dialog/cancel-query-dialog';
@ -114,6 +114,8 @@ LIMIT 100`,
queryManager.rerunLastQuery(true);
}, 30000);
const now = useClock();
const incrementWorkVersion = useWorkStateStore(state => state.increment);
const queryTaskHistory = queryTaskHistoryState.getSomeData();
@ -205,6 +207,11 @@ LIMIT 100`,
</Menu>
);
const duration =
w.taskStatus === 'RUNNING'
? now.valueOf() - new Date(w.createdTime).valueOf()
: w.duration;
const [icon, color] = statusToIconAndColor(w.taskStatus);
return (
<Popover2 className="work-entry" key={w.taskId} position="left" content={menu}>
@ -217,7 +224,7 @@ LIMIT 100`,
/>
<div className="timing">
{w.createdTime.replace('T', ' ').replace(/\.\d\d\dZ$/, '') +
(w.duration > 0 ? ` (${formatDuration(w.duration)})` : '')}
(duration > 0 ? ` (${formatDuration(duration)})` : '')}
</div>
</div>
<div className="line2">

10
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -91,11 +91,13 @@ export interface RunPanelProps {
small?: boolean;
onRun(preview: boolean): void;
queryEngines: DruidEngine[];
clusterCapacity: number | undefined;
moreMenu?: JSX.Element;
}
export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines } = props;
const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines, clusterCapacity } =
props;
const [editContextDialogOpen, setEditContextDialogOpen] = useState(false);
const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false);
const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState<IndexSpec | undefined>();
@ -375,7 +377,11 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
/>
</Popover2>
{effectiveEngine === 'sql-msq-task' && (
<MaxTasksButton queryContext={queryContext} changeQueryContext={changeQueryContext} />
<MaxTasksButton
clusterCapacity={clusterCapacity}
queryContext={queryContext}
changeQueryContext={changeQueryContext}
/>
)}
</ButtonGroup>
)}

12
web-console/src/views/workbench-view/workbench-view.tsx
View File

@ -33,7 +33,12 @@ import {
TabEntry,
WorkbenchQuery,
} from '../../druid-models';
import { convertSpecToSql, getSpecDatasourceName, getTaskExecution } from '../../helpers';
import {
Capabilities,
convertSpecToSql,
getSpecDatasourceName,
getTaskExecution,
} from '../../helpers';
import { getLink } from '../../links';
import { AppToaster } from '../../singletons';
import { AceEditorStateCache } from '../../singletons/ace-editor-state-cache';
@ -80,6 +85,7 @@ function externalDataTabId(tabId: string | undefined): boolean {
}
export interface WorkbenchViewProps {
capabilities: Capabilities;
tabId: string | undefined;
onTabChange(newTabId: string): void;
initQueryWithContext: QueryWithContext | undefined;
@ -628,7 +634,8 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
}
private renderCenterPanel() {
const { mandatoryQueryContext, queryEngines, allowExplain, goToIngestion } = this.props;
const { capabilities, mandatoryQueryContext, queryEngines, allowExplain, goToIngestion } =
this.props;
const { columnMetadataState } = this.state;
const currentTabEntry = this.getCurrentTabEntry();
@ -647,6 +654,7 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
onQueryTab={this.handleNewTab}
onDetails={this.handleDetails}
queryEngines={queryEngines}
clusterCapacity={capabilities.getClusterCapacity()}
goToIngestion={goToIngestion}
runMoreMenu={
<Menu>