Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: Fix order-by-delta in explore view table (#16417) 

* change to using measure name

* Implment order by delta

* less paring, stricter types

* safeDivide0

* fix no query

* new DTQ alows parsing JSON_VALUE(...RETURNING...)
This commit is contained in:
Vadim Ogievetsky 2024-05-13 19:03:46 -07:00 committed by GitHub
parent d1100a6f63
commit 760e449875
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 556 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
licenses.yaml
View File

@ -5094,7 +5094,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Imply Data
version: 0.22.11
version: 0.22.13
---

14
web-console/package-lock.json generated
View File

@ -14,7 +14,7 @@
"@blueprintjs/datetime2": "^0.9.35",
"@blueprintjs/icons": "^4.16.0",
"@blueprintjs/popover2": "^1.14.9",
"@druid-toolkit/query": "^0.22.11",
"@druid-toolkit/query": "^0.22.13",
"@druid-toolkit/visuals-core": "^0.3.3",
"@druid-toolkit/visuals-react": "^0.3.3",
"ace-builds": "~1.4.14",
@ -1004,9 +1004,9 @@
}
},
"node_modules/@druid-toolkit/query": {
"version": "0.22.11",
"resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.11.tgz",
"integrity": "sha512-VVEn/tsEr9fb+8eKc+nu3/YH7l+LZ1vd0D32UDo66GLS3cI+EKOCM7VYC8lTvB1tAS+98w/EzfbdlRPlkSeOoQ==",
"version": "0.22.13",
"resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.13.tgz",
"integrity": "sha512-p0Cmmbk55vLaYs2WWcUr09qDRU2IrkXOxGgUG+wS6Uuq/ALBqSmUDlbMSxB3vJjMvegiwgJ8+n7VfVpO0t/bJg==",
"dependencies": {
"tslib": "^2.5.2"
}
@ -19146,9 +19146,9 @@
"dev": true
},
"@druid-toolkit/query": {
"version": "0.22.11",
"resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.11.tgz",
"integrity": "sha512-VVEn/tsEr9fb+8eKc+nu3/YH7l+LZ1vd0D32UDo66GLS3cI+EKOCM7VYC8lTvB1tAS+98w/EzfbdlRPlkSeOoQ==",
"version": "0.22.13",
"resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.13.tgz",
"integrity": "sha512-p0Cmmbk55vLaYs2WWcUr09qDRU2IrkXOxGgUG+wS6Uuq/ALBqSmUDlbMSxB3vJjMvegiwgJ8+n7VfVpO0t/bJg==",
"requires": {
"tslib": "^2.5.2"
}

2
web-console/package.json
View File

@ -68,7 +68,7 @@
"@blueprintjs/datetime2": "^0.9.35",
"@blueprintjs/icons": "^4.16.0",
"@blueprintjs/popover2": "^1.14.9",
"@druid-toolkit/query": "^0.22.11",
"@druid-toolkit/query": "^0.22.13",
"@druid-toolkit/visuals-core": "^0.3.3",
"@druid-toolkit/visuals-react": "^0.3.3",
"ace-builds": "~1.4.14",

2
web-console/src/components/record-table-pane/record-table-pane.tsx
View File

@ -104,7 +104,7 @@ export const RecordTablePane = React.memo(function RecordTablePane(props: Record
const finalPage =
hasMoreResults && Math.floor(queryResult.rows.length / pagination.pageSize) === pagination.page; // on the last page
const numericColumnBraces = getNumericColumnBraces(queryResult, pagination);
const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, pagination);
return (
<div className={classNames('record-table-pane', { 'more-results': hasMoreResults })}>
{finalPage ? (

4
web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx
View File

@ -38,8 +38,8 @@ export interface AsyncActionDialogProps {
className?: string;
icon?: IconName;
intent?: Intent;
successText: string;
failText: string;
successText: ReactNode;
failText: ReactNode;
warningChecks?: ReactNode[];
children?: ReactNode;
}

7
web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
View File

@ -66,7 +66,12 @@ export const KillDatasourceDialog = function KillDatasourceDialog(
return resp.data;
}}
confirmButtonText="Permanently delete unused segments"
successText="Kill task was issued. Unused segments in datasource will be deleted"
successText={
<>
Kill task was issued. Unused segments in datasource <Tag minimal>{datasource}</Tag> will
be deleted
</>
}
failText="Failed submit kill task"
intent={Intent.DANGER}
onClose={onClose}

10
web-console/src/druid-models/execution/execution.ts
View File

@ -440,7 +440,10 @@ export class Execution {
value.queryContext = queryContext;
const parsedQuery = parseSqlQuery(sqlQuery);
if (value.result && (parsedQuery || queryContext)) {
value.result = value.result.attachQuery({ context: queryContext }, parsedQuery);
value.result = value.result.attachQuery(
{ ...this.nativeQuery, context: queryContext },
parsedQuery,
);
}
return new Execution(value);
@ -463,7 +466,10 @@ export class Execution {
public changeResult(result: QueryResult): Execution {
return new Execution({
...this.valueOf(),
result: result.attachQuery({}, this.sqlQuery ? parseSqlQuery(this.sqlQuery) : undefined),
result: result.attachQuery(
this.nativeQuery,
this.sqlQuery ? parseSqlQuery(this.sqlQuery) : undefined,
),
});
}

5
web-console/src/druid-models/workbench-query/workbench-query.ts
View File

@ -18,6 +18,7 @@
import type {
QueryParameter,
QueryPayload,
SqlClusteredByClause,
SqlExpression,
SqlPartitionedByClause,
@ -446,7 +447,7 @@ export class WorkbenchQuery {
public getApiQuery(makeQueryId: () => string = uuidv4): {
engine: DruidEngine;
query: Record<string, any>;
query: QueryPayload;
prefixLines: number;
cancelQueryId?: string;
} {
@ -478,7 +479,7 @@ export class WorkbenchQuery {
};
}
let apiQuery: Record<string, any> = {};
let apiQuery: QueryPayload;
if (this.isJsonLike()) {
try {
apiQuery = Hjson.parse(queryString);

16
web-console/src/utils/general.tsx
View File

@ -338,6 +338,22 @@ export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string)
// ----------------------------
export function partition<T>(xs: T[], predicate: (x: T, i: number) => boolean): [T[], T[]] {
const match: T[] = [];
const nonMatch: T[] = [];
for (let i = 0; i < xs.length; i++) {
const x = xs[i];
if (predicate(x, i)) {
match.push(x);
} else {
nonMatch.push(x);
}
}
return [match, nonMatch];
}
export function filterMap<T, Q>(xs: readonly T[], f: (x: T, i: number) => Q | undefined): Q[] {
return xs.map(f).filter((x: Q | undefined) => typeof x !== 'undefined') as Q[];
}

12
web-console/src/utils/table-helpers.ts
View File

@ -32,9 +32,16 @@ export function changePage(pagination: Pagination, page: number): Pagination {
return deepSet(pagination, 'page', page);
}
export interface ColumnHint {
displayName?: string;
group?: string;
formatter?: (x: any) => string;
}
export function getNumericColumnBraces(
queryResult: QueryResult,
pagination?: Pagination,
columnHints: Map<string, ColumnHint> | undefined,
pagination: Pagination | undefined,
): Record<number, string[]> {
let rows = queryResult.rows;
@ -47,8 +54,9 @@ export function getNumericColumnBraces(
if (rows.length) {
queryResult.header.forEach((column, i) => {
if (!oneOf(column.nativeType, 'LONG', 'FLOAT', 'DOUBLE')) return;
const formatter = columnHints?.get(column.name)?.formatter || formatNumber;
const brace = filterMap(rows, row =>
oneOf(typeof row[i], 'number', 'bigint') ? formatNumber(row[i]) : undefined,
oneOf(typeof row[i], 'number', 'bigint') ? formatter(row[i]) : undefined,
);
if (rows.length === brace.length) {
numericColumnBraces[i] = brace;

31
web-console/src/views/datasources-view/datasources-view.tsx
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
import { FormGroup, InputGroup, Intent, MenuItem, Switch } from '@blueprintjs/core';
import { FormGroup, InputGroup, Intent, MenuItem, Switch, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { SqlQuery, T } from '@druid-toolkit/query';
import classNames from 'classnames';
@ -651,8 +651,18 @@ GROUP BY 1, 2`;
return resp.data;
}}
confirmButtonText="Mark as unused all segments"
successText="All segments in datasource have been marked as unused"
failText="Failed to mark as unused all segments in datasource"
successText={
<>
All segments in datasource <Tag minimal>{datasourceToMarkAsUnusedAllSegmentsIn}</Tag>{' '}
have been marked as unused
</>
}
failText={
<>
Failed to mark as unused all segments in datasource{' '}
<Tag minimal>{datasourceToMarkAsUnusedAllSegmentsIn}</Tag>
</>
}
intent={Intent.DANGER}
onClose={() => {
this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: undefined });
@ -684,8 +694,19 @@ GROUP BY 1, 2`;
return resp.data;
}}
confirmButtonText="Mark as used all segments"
successText="All non-overshadowed segments in datasource have been marked as used"
failText="Failed to mark as used all non-overshadowed segments in datasource"
successText={
<>
All non-overshadowed segments in datasource{' '}
<Tag minimal>{datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn}</Tag> have been marked
as used
</>
}
failText={
<>
Failed to mark as used all non-overshadowed segments in datasource{' '}
<Tag minimal>{datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn}</Tag>
</>
}
intent={Intent.PRIMARY}
onClose={() => {
this.setState({ datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: undefined });

56
web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.tsx
View File

@ -31,7 +31,7 @@ import ReactTable from 'react-table';
import { BracedText, Deferred, TableCell } from '../../../../../components';
import { possibleDruidFormatForValues, TIME_COLUMN } from '../../../../../druid-models';
import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../../../react-table';
import type { Pagination, QueryAction } from '../../../../../utils';
import type { ColumnHint, Pagination, QueryAction } from '../../../../../utils';
import {
columnToIcon,
columnToWidth,
@ -60,30 +60,34 @@ function isComparable(x: unknown): boolean {
return x !== null && x !== '';
}
function columnNester(columns: TableColumn[], groupHints: string[] | undefined): TableColumn[] {
if (!groupHints) return columns;
function columnNester(
tableColumns: TableColumn[],
resultColumns: readonly Column[],
columnHints: Map<string, ColumnHint> | undefined,
): TableColumn[] {
if (!columnHints) return tableColumns;
const ret: TableColumn[] = [];
let currentGroupHint: string | null = null;
let currentGroupName: string | null = null;
let currentColumnGroup: TableColumn | null = null;
for (let i = 0; i < columns.length; i++) {
const column = columns[i];
const groupHint = groupHints[i];
if (groupHint) {
if (currentGroupHint === groupHint) {
currentColumnGroup!.columns!.push(column);
for (let i = 0; i < tableColumns.length; i++) {
const tableColumn = tableColumns[i];
const group = columnHints.get(resultColumns[i].name)?.group;
if (group) {
if (currentGroupName === group) {
currentColumnGroup!.columns!.push(tableColumn);
} else {
currentGroupHint = groupHint;
currentGroupName = group;
ret.push(
(currentColumnGroup = {
Header: <div className="group-cell">{currentGroupHint}</div>,
columns: [column],
Header: <div className="group-cell">{currentGroupName}</div>,
columns: [tableColumn],
}),
);
}
} else {
ret.push(column);
currentGroupHint = null;
ret.push(tableColumn);
currentGroupName = null;
currentColumnGroup = null;
}
}
@ -94,12 +98,12 @@ function columnNester(columns: TableColumn[], groupHints: string[] | undefined):
export interface GenericOutputTableProps {
queryResult: QueryResult;
onQueryAction(action: QueryAction): void;
onOrderByChange?(columnIndex: number, desc: boolean): void;
onOrderByChange?(columnName: string, desc: boolean): void;
onExport?(): void;
runeMode: boolean;
showTypeIcons: boolean;
initPageSize?: number;
groupHints?: string[];
columnHints?: Map<string, ColumnHint>;
}
export const GenericOutputTable = React.memo(function GenericOutputTable(
@ -113,7 +117,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
runeMode,
showTypeIcons,
initPageSize,
groupHints,
columnHints,
} = props;
const parsedQuery = queryResult.sqlQuery;
const [pagination, setPagination] = useState<Pagination>({
@ -159,7 +163,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
icon={reverseOrderByDirection === 'ASC' ? IconNames.SORT_ASC : IconNames.SORT_DESC}
text={`Order ${reverseOrderByDirection === 'ASC' ? 'ascending' : 'descending'}`}
onClick={() => {
onOrderByChange(headerIndex, reverseOrderByDirection !== 'ASC');
onOrderByChange(header, reverseOrderByDirection !== 'ASC');
}}
/>,
);
@ -170,7 +174,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
icon={IconNames.SORT_DESC}
text="Order descending"
onClick={() => {
onOrderByChange(headerIndex, true);
onOrderByChange(header, true);
}}
/>,
<MenuItem
@ -178,7 +182,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
icon={IconNames.SORT_ASC}
text="Order ascending"
onClick={() => {
onOrderByChange(headerIndex, false);
onOrderByChange(header, false);
}}
/>,
);
@ -426,7 +430,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
const finalPage =
hasMoreResults && Math.floor(queryResult.rows.length / pagination.pageSize) === pagination.page; // on the last page
const numericColumnBraces = getNumericColumnBraces(queryResult, pagination);
const numericColumnBraces = getNumericColumnBraces(queryResult, columnHints, pagination);
return (
<div className={classNames('generic-output-table', { 'more-results': hasMoreResults })}>
{finalPage ? (
@ -479,7 +483,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
<div className="clickable-cell">
<div className="output-name">
{icon && <Icon className="type-icon" icon={icon} size={12} />}
{h}
{columnHints?.get(h)?.displayName ?? h}
{hasFilterOnHeader(h, i) && <Icon icon={IconNames.FILTER} size={14} />}
</div>
</div>
@ -490,6 +494,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
accessor: String(i),
Cell(row) {
const value = row.value;
const formatter = columnHints?.get(h)?.formatter || formatNumber;
return (
<div>
<Popover2
@ -498,7 +503,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
{numericColumnBraces[i] ? (
<BracedText
className="table-padding"
text={formatNumber(value)}
text={formatter(value)}
braces={numericColumnBraces[i]}
padFractionalPart
/>
@ -516,7 +521,8 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
: undefined,
};
}),
groupHints,
queryResult.header,
columnHints,
)}
/>
)}

417
web-console/src/views/explore-view/modules/table-react-module.tsx
View File

@ -16,11 +16,13 @@
* limitations under the License.
*/
import type { SqlOrderByExpression } from '@druid-toolkit/query';
import { Button } from '@blueprintjs/core';
import type { SqlOrderByExpression, SqlTable } from '@druid-toolkit/query';
import {
C,
F,
SqlCase,
SqlColumn,
SqlExpression,
SqlFunction,
SqlLiteral,
@ -35,15 +37,23 @@ import ReactDOM from 'react-dom';
import { Loader } from '../../../components';
import { useQueryManager } from '../../../hooks';
import type { ColumnHint } from '../../../utils';
import { formatInteger, formatPercent } from '../../../utils';
import { getInitQuery } from '../utils';
import { GenericOutputTable } from './components';
import { shiftTimeInWhere } from './utils/utils';
import { getWhereForCompares, shiftTimeInExpression } from './utils/utils';
import './table-react-module.scss';
type MultipleValueMode = 'null' | 'empty' | 'latest' | 'latestNonNull' | 'count';
type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPercent';
// As of this writing ordering the outer query on something other than __time sometimes throws an error, set this to false / remove it
// when ordering on non __time is more robust
const NEEDS_GROUPING_TO_ORDER = true;
const KNOWN_AGGREGATIONS = [
'COUNT',
'SUM',
@ -73,13 +83,37 @@ const KNOWN_AGGREGATIONS = [
'ANY_VALUE',
];
const TOP_VALUES_NAME = 'top_values';
const TOP_VALUES_K = 5000;
function coalesce0(ex: SqlExpression) {
return F('COALESCE', ex, SqlLiteral.ZERO);
}
function safeDivide0(a: SqlExpression, b: SqlExpression) {
return coalesce0(F('SAFE_DIVIDE', a, b));
}
function anyValue(ex: SqlExpression) {
return F('ANY_VALUE', ex);
}
function addTableScope(expression: SqlExpression, newTableScope: string): SqlExpression {
return expression.walk(ex => {
if (ex instanceof SqlColumn && !ex.getTableName()) {
return ex.changeTableName(newTableScope);
}
return ex;
}) as SqlExpression;
}
function toGroupByExpression(
splitColumn: ExpressionMeta,
timeBucket: string,
compareShiftDuration?: string,
) {
const { expression, sqlType, name } = splitColumn;
return expression
return addTableScope(expression, 't')
.applyIf(sqlType === 'TIMESTAMP' && compareShiftDuration, e =>
F.timeShift(e, compareShiftDuration!, 1),
)
@ -131,9 +165,21 @@ function toShowColumnExpression(
return ex.as(showColumn.name);
}
function getJoinCondition(
splitColumns: ExpressionMeta[],
table1: SqlTable,
table2: SqlTable,
): SqlExpression {
return SqlExpression.and(
...splitColumns.map(splitColumn =>
table1.column(splitColumn.name).isNotDistinctFrom(table2.column(splitColumn.name)),
),
);
}
interface QueryAndHints {
query: SqlQuery;
groupHints: string[];
columnHints: Map<string, ColumnHint>;
}
export default typedVisualModule({
@ -200,13 +246,14 @@ export default typedVisualModule({
compares: {
type: 'options',
options: ['PT1M', 'PT5M', 'PT1H', 'P1D', 'P1M'],
options: ['PT1M', 'PT5M', 'PT1H', 'PT6H', 'P1D', 'P1M'],
control: {
label: 'Compares',
optionLabels: {
PT1M: '1 minute',
PT5M: '5 minutes',
PT1H: '1 hour',
PT6H: '6 hours',
P1D: '1 day',
P1M: '1 month',
},
@ -214,12 +261,31 @@ export default typedVisualModule({
},
},
showDelta: {
type: 'boolean',
compareTypes: {
type: 'options',
options: ['value', 'delta', 'absDelta', 'percent', 'absPercent'],
default: ['value', 'delta'],
control: {
visible: ({ params }) => Boolean((params.compares || []).length),
label: 'Compare types',
visible: ({ params }) => Boolean((params.compares || []).length) && !params.pivotColumn,
optionLabels: {
value: 'Value',
delta: 'Delta',
absDelta: 'Abs. delta',
percent: 'Percent',
absPercent: 'Abs. percent',
},
},
},
restrictTop: {
type: 'boolean',
default: true,
control: {
label: `Restrict to top ${formatInteger(TOP_VALUES_K)} when ordering on delta`,
visible: ({ params }) => Boolean((params.compares || []).length) && !params.pivotColumn,
},
},
maxRows: {
type: 'number',
default: 200,
@ -287,7 +353,7 @@ function TableModule(props: TableModuleProps) {
},
});
const queryAndHints = useMemo(() => {
const queryAndHints = useMemo((): QueryAndHints | undefined => {
const splitColumns: ExpressionMeta[] = parameterValues.splitColumns;
const timeBucket: string = parameterValues.timeBucket || 'PT1H';
const showColumns: ExpressionMeta[] = parameterValues.showColumns;
@ -295,26 +361,71 @@ function TableModule(props: TableModuleProps) {
const pivotColumn: ExpressionMeta = parameterValues.pivotColumn;
const metrics: ExpressionMeta[] = parameterValues.metrics;
const compares: string[] = parameterValues.compares || [];
const showDelta: boolean = parameterValues.showDelta;
const compareTypes: CompareType[] = parameterValues.compareTypes;
const restrictTop: boolean = parameterValues.restrictTop;
const maxRows: number = parameterValues.maxRows;
const pivotValues = pivotColumn ? pivotValueState.data : undefined;
if (pivotColumn && !pivotValues) return;
const hasCompare = Boolean(compares.length);
const effectiveOrderBy =
orderBy || C(metrics[0]?.name || splitColumns[0]?.name).toOrderByExpression('DESC');
const hasCompare = !pivotColumn && Boolean(compares.length) && Boolean(compareTypes.length);
const orderByColumnName = (effectiveOrderBy.expression as SqlColumn).getName();
let orderByCompareMeasure: string | undefined;
let orderByCompareDuration: string | undefined;
let orderByCompareType: CompareType | undefined;
if (hasCompare) {
const m = orderByColumnName.match(
/^(.+):cmp:([^:]+):(value|delta|absDelta|percent|absPercent)$/,
);
if (m) {
orderByCompareMeasure = m[1];
orderByCompareDuration = m[2];
orderByCompareType = m[3] as CompareType;
}
}
const metricExpression = metrics.find(m => m.name === orderByCompareMeasure)?.expression;
const topValuesQuery =
restrictTop && metricExpression && orderByCompareType !== 'value' && splitColumns.length
? getInitQuery(table, getWhereForCompares(where, compares))
.applyForEach(splitColumns, (q, splitColumn) =>
q.addSelect(toGroupByExpression(splitColumn, timeBucket), {
addToGroupBy: 'end',
}),
)
.changeOrderByExpression(metricExpression.toOrderByExpression('DESC'))
.changeLimitValue(TOP_VALUES_K)
: undefined;
const columnHints = new Map<string, ColumnHint>();
const mainQuery = getInitQuery(table, where)
.applyIf(topValuesQuery, q =>
q.addInnerJoin(
T(TOP_VALUES_NAME),
getJoinCondition(splitColumns, T('t'), T(TOP_VALUES_NAME)),
),
)
.applyForEach(splitColumns, (q, splitColumn) =>
q.addSelect(toGroupByExpression(splitColumn, timeBucket), {
addToGroupBy: 'end',
}),
)
.applyForEach(showColumns, (q, showColumn) =>
q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)),
.applyIf(!orderByCompareDuration, q =>
q.applyForEach(showColumns, (q, showColumn) =>
q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)),
),
)
.applyForEach(pivotValues || [''], (q, pivotValue, i) =>
q.applyForEach(metrics, (q, metric) =>
q.addSelect(
q.applyForEach(metrics, (q, metric) => {
const alias = `${metric.name}${pivotColumn && i > 0 ? `:${pivotValue}` : ''}`;
if (pivotColumn) {
columnHints.set(alias, { displayName: metric.name, group: pivotValue });
}
return q.addSelect(
metric.expression
.as(metric.name)
.applyIf(pivotColumn, q =>
@ -323,115 +434,204 @@ function TableModule(props: TableModuleProps) {
pivotColumn.expression.equal(pivotValue),
KNOWN_AGGREGATIONS,
)
.as(`${metric.name}${i > 0 ? ` [${pivotValue}]` : ''}`),
.as(alias),
),
),
),
);
}),
)
.applyIf(metrics.length > 0 || splitColumns.length > 0, q =>
q.changeOrderByExpression(
orderBy || C(metrics[0]?.name || splitColumns[0]?.name).toOrderByExpression('DESC'),
),
)
.changeLimitValue(maxRows);
.applyIf(!orderByCompareDuration, q =>
q
.applyIf(metrics.length > 0 || splitColumns.length > 0, q =>
q.changeOrderByExpression(effectiveOrderBy),
)
.changeLimitValue(maxRows),
);
if (!hasCompare) {
return {
query: mainQuery,
groupHints: pivotColumn
? splitColumns
.map(() => '')
.concat(
showColumns.map(() => ''),
(pivotValues || []).flatMap(v => metrics.map(() => v)),
)
: [],
columnHints,
};
}
const main = T('main');
return {
query: SqlQuery.from(main)
.changeWithParts(
[SqlWithPart.simple('main', mainQuery)].concat(
compares.map((comparePeriod, i) =>
SqlWithPart.simple(
`compare${i}`,
getInitQuery(table, shiftTimeInWhere(where, comparePeriod))
.applyForEach(splitColumns, (q, splitColumn) =>
q.addSelect(toGroupByExpression(splitColumn, timeBucket, comparePeriod), {
addToGroupBy: 'end',
}),
)
.applyForEach(metrics, (q, metric) =>
q.addSelect(metric.expression.as(metric.name)),
const leader = T(orderByCompareDuration ? `compare_${orderByCompareDuration}` : 'main');
const query = SqlQuery.from(leader)
.changeWithParts(
(
(topValuesQuery
? [SqlWithPart.simple(TOP_VALUES_NAME, topValuesQuery)]
: []) as SqlWithPart[]
).concat(
SqlWithPart.simple('main', mainQuery),
compares.map(compare =>
SqlWithPart.simple(
`compare_${compare}`,
getInitQuery(table, shiftTimeInExpression(where, compare))
.applyIf(topValuesQuery, q =>
q.addInnerJoin(
T(TOP_VALUES_NAME),
getJoinCondition(splitColumns, T('t'), T(TOP_VALUES_NAME)),
),
),
)
.applyForEach(splitColumns, (q, splitColumn) =>
q.addSelect(toGroupByExpression(splitColumn, timeBucket, compare), {
addToGroupBy: 'end',
}),
)
.applyIf(orderByCompareDuration === compare, q =>
q.applyForEach(showColumns, (q, showColumn) =>
q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)),
),
)
.applyForEach(metrics, (q, metric) =>
q.addSelect(metric.expression.as(metric.name)),
)
.applyIf(compare === orderByCompareDuration && orderByCompareType === 'value', q =>
q
.changeOrderByExpression(
effectiveOrderBy.changeExpression(C(orderByCompareMeasure!)),
)
.changeLimitValue(maxRows),
),
),
),
)
.changeSelectExpressions(
splitColumns
.map(splitColumn => main.column(splitColumn.name).as(splitColumn.name))
.concat(
showColumns.map(showColumn => main.column(showColumn.name).as(showColumn.name)),
metrics.map(metric => main.column(metric.name).as(metric.name)),
compares.flatMap((_, i) =>
metrics.flatMap(metric => {
const c = T(`compare${i}`).column(metric.name);
const ret = [SqlFunction.simple('COALESCE', [c, 0]).as(`#prev: ${metric.name}`)];
if (showDelta) {
ret.push(
F.stringFormat(
'%.1f%%',
SqlFunction.simple('SAFE_DIVIDE', [
SqlExpression.parse(`(${main.column(metric.name)} - ${c}) * 100.0`),
c,
]),
).as(`%chg: ${metric.name}`),
);
}
return ret;
}),
),
),
)
.changeSelectExpressions(
splitColumns
.map(splitColumn => main.column(splitColumn.name).as(splitColumn.name))
.concat(
showColumns.map(showColumn =>
leader
.column(showColumn.name)
.applyIf(NEEDS_GROUPING_TO_ORDER, anyValue)
.as(showColumn.name),
),
)
.applyForEach(compares, (q, _comparePeriod, i) =>
metrics.map(metric =>
main
.column(metric.name)
.applyIf(NEEDS_GROUPING_TO_ORDER, anyValue)
.applyIf(orderByCompareDuration, coalesce0)
.as(metric.name),
),
compares.flatMap(compare =>
metrics.flatMap(metric => {
const c = T(`compare_${compare}`)
.column(metric.name)
.applyIf(NEEDS_GROUPING_TO_ORDER, anyValue)
.applyIf(compare !== orderByCompareDuration, coalesce0);
const mainMetric = main
.column(metric.name)
.applyIf(NEEDS_GROUPING_TO_ORDER, anyValue)
.applyIf(orderByCompareDuration, coalesce0);
const diff = mainMetric.subtract(c);
const ret: SqlExpression[] = [];
if (compareTypes.includes('value')) {
const valueName = `${metric.name}:cmp:${compare}:value`;
columnHints.set(valueName, {
group: `Comparison to ${compare}`,
displayName: `${metric.name} (value)`,
});
ret.push(c.as(valueName));
}
if (compareTypes.includes('delta')) {
const deltaName = `${metric.name}:cmp:${compare}:delta`;
columnHints.set(deltaName, {
group: `Comparison to ${compare}`,
displayName: `${metric.name} (delta)`,
});
ret.push(diff.as(deltaName));
}
if (compareTypes.includes('absDelta')) {
const deltaName = `${metric.name}:cmp:${compare}:absDelta`;
columnHints.set(deltaName, {
group: `Comparison to ${compare}`,
displayName: `${metric.name} (Abs. delta)`,
});
ret.push(F('ABS', diff).as(deltaName));
}
if (compareTypes.includes('percent')) {
const percentName = `${metric.name}:cmp:${compare}:percent`;
columnHints.set(percentName, {
group: `Comparison to ${compare}`,
displayName: `${metric.name} (%)`,
formatter: formatPercent,
});
ret.push(
safeDivide0(diff.multiply(SqlLiteral.ONE_POINT_ZERO), c).as(percentName),
);
}
if (compareTypes.includes('absPercent')) {
const percentName = `${metric.name}:cmp:${compare}:absPercent`;
columnHints.set(percentName, {
group: `Comparison to ${compare}`,
displayName: `${metric.name} (abs. %)`,
formatter: formatPercent,
});
ret.push(
F('ABS', safeDivide0(diff.multiply(SqlLiteral.ONE_POINT_ZERO), c)).as(
percentName,
),
);
}
return ret;
}),
),
),
)
.applyIf(orderByCompareDuration, q =>
q.addLeftJoin(
main,
getJoinCondition(splitColumns, main, T(`compare_${orderByCompareDuration}`)),
),
)
.applyForEach(
compares.filter(c => c !== orderByCompareDuration),
(q, compare) =>
q.addLeftJoin(
T(`compare${i}`),
SqlExpression.and(
...splitColumns.map(splitColumn =>
main
.column(splitColumn.name)
.isNotDistinctFrom(T(`compare${i}`).column(splitColumn.name)),
),
),
T(`compare_${compare}`),
getJoinCondition(splitColumns, main, T(`compare_${compare}`)),
),
),
groupHints: splitColumns
.map(() => 'Current')
.concat(
showColumns.map(() => 'Current'),
metrics.map(() => 'Current'),
compares.flatMap(comparePeriod =>
metrics
.flatMap(() => (showDelta ? ['', ''] : ['']))
.map(() => `Comparison to ${comparePeriod}`),
),
),
)
.applyIf(NEEDS_GROUPING_TO_ORDER, q =>
q.changeGroupByExpressions(splitColumns.map((_, i) => SqlLiteral.index(i))),
)
.addOrderBy(effectiveOrderBy)
.changeLimitValue(maxRows);
for (const splitColumn of splitColumns) {
columnHints.set(splitColumn.name, { group: 'Current' });
}
for (const showColumn of showColumns) {
columnHints.set(showColumn.name, { group: 'Current' });
}
for (const metric of metrics) {
columnHints.set(metric.name, { group: 'Current' });
}
return {
query,
columnHints,
};
}, [table, where, parameterValues, orderBy, pivotValueState.data]);
const [resultState] = useQueryManager({
query: queryAndHints,
processQuery: async (queryAndHints: QueryAndHints) => {
const { query, groupHints } = queryAndHints;
const { query, columnHints } = queryAndHints;
return {
result: await sqlQuery(query),
groupHints,
columnHints,
};
},
});
@ -440,19 +640,24 @@ function TableModule(props: TableModuleProps) {
return (
<div className="table-module">
{resultState.error ? (
resultState.getErrorMessage()
<div>
<div>{resultState.getErrorMessage()}</div>
{resultState.getErrorMessage()?.includes('not found in any table') && orderBy && (
<Button text="Clear order by" onClick={() => setOrderBy(undefined)} />
)}
</div>
) : resultData ? (
<GenericOutputTable
runeMode={false}
queryResult={resultData.result}
groupHints={resultData.groupHints}
columnHints={resultData.columnHints}
showTypeIcons={false}
onOrderByChange={(headerIndex, desc) => {
const idx = SqlLiteral.index(headerIndex);
if (orderBy && String(orderBy.expression) === String(idx)) {
onOrderByChange={(columnName, desc) => {
const column = C(columnName);
if (orderBy && orderBy.expression.equals(column)) {
setOrderBy(orderBy.reverseDirection());
} else {
setOrderBy(idx.toOrderByExpression(desc ? 'DESC' : 'ASC'));
setOrderBy(column.toOrderByExpression(desc ? 'DESC' : 'ASC'));
}
}}
onQueryAction={action => {

57
web-console/src/views/explore-view/modules/utils/utils.spec.ts
View File

@ -18,21 +18,64 @@
import { SqlExpression } from '@druid-toolkit/query';
import { shiftTimeInWhere } from './utils';
import { getWhereForCompares, shiftTimeInExpression } from './utils';
describe('shiftTimeInWhere', () => {
it('works with TIME_IN_INTERVAL', () => {
describe('getWhereForCompares', () => {
it('works', () => {
expect(
shiftTimeInWhere(
getWhereForCompares(
SqlExpression.parse(
`TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28') AND "country" = 'United States'`,
),
['PT1H', 'P1D'],
).toString(),
).toEqual(
`(TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28') OR (TIME_SHIFT(TIMESTAMP '2016-06-27', 'PT1H', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'PT1H', -1)) OR (TIME_SHIFT(TIMESTAMP '2016-06-27', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'P1D', -1))) AND "country" = 'United States'`,
);
});
});
describe('shiftTimeInExpression', () => {
it('works with TIME_IN_INTERVAL (date)', () => {
expect(
shiftTimeInExpression(
SqlExpression.parse(`TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28')`),
'P1D',
).toString(),
).toEqual(`TIME_IN_INTERVAL(TIME_SHIFT("__time", 'P1D', 1), '2016-06-27/2016-06-28')`);
).toEqual(
`TIME_SHIFT(TIMESTAMP '2016-06-27', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'P1D', -1)`,
);
});
it('works with TIME_IN_INTERVAL (date and time)', () => {
expect(
shiftTimeInExpression(
SqlExpression.parse(
`TIME_IN_INTERVAL("__time", '2016-06-27T12:34:56/2016-06-28T12:34:56')`,
),
'P1D',
).toString(),
).toEqual(
`TIME_SHIFT(TIMESTAMP '2016-06-27 12:34:56', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28 12:34:56', 'P1D', -1)`,
);
});
it('works with TIME_IN_INTERVAL (date and time, zulu)', () => {
expect(
shiftTimeInExpression(
SqlExpression.parse(
`TIME_IN_INTERVAL("__time", '2016-06-27T12:34:56Z/2016-06-28T12:34:56Z')`,
),
'P1D',
).toString(),
).toEqual(
`TIME_SHIFT(TIME_PARSE('2016-06-27 12:34:56', NULL, 'Etc/UTC'), 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIME_PARSE('2016-06-28 12:34:56', NULL, 'Etc/UTC'), 'P1D', -1)`,
);
});
it('works with relative time', () => {
expect(
shiftTimeInWhere(
shiftTimeInExpression(
SqlExpression.parse(
`(TIME_SHIFT(MAX_DATA_TIME(), 'PT1H', -1) <= "__time" AND "__time" < MAX_DATA_TIME())`,
),
@ -45,7 +88,7 @@ describe('shiftTimeInWhere', () => {
it('works with relative time (specific timestamps)', () => {
expect(
shiftTimeInWhere(
shiftTimeInExpression(
SqlExpression.parse(
`TIMESTAMP '2016-06-27 20:31:02.498' <= "__time" AND "__time" < TIMESTAMP '2016-06-27 21:31:02.498'`,
),

63
web-console/src/views/explore-view/modules/utils/utils.ts
View File

@ -16,28 +16,77 @@
* limitations under the License.
*/
import type { SqlExpression } from '@druid-toolkit/query';
import { F, SqlFunction, SqlLiteral } from '@druid-toolkit/query';
import { F, SqlExpression, SqlFunction, SqlLiteral } from '@druid-toolkit/query';
export function shiftTimeInWhere(where: SqlExpression, period: string): SqlExpression {
return where.walk(ex => {
import { partition } from '../../../../utils';
const IS_DATE_LIKE = /^[+-]?\d\d\d\d[^']+$/;
function isoStringToTimestampLiteral(iso: string): SqlExpression {
const zulu = iso.endsWith('Z');
const cleanIso = iso.replace('T', ' ').replace('Z', '');
let sql: string;
if (zulu) {
sql = `TIME_PARSE('${cleanIso}', NULL, 'Etc/UTC')`;
} else {
sql = `TIMESTAMP '${cleanIso}'`;
}
return SqlExpression.parse(sql);
}
export function getWhereForCompares(where: SqlExpression, compares: string[]): SqlExpression {
const whereParts = where.decomposeViaAnd({ flatten: true });
const [timeExpressions, timelessExpressions] = partition(whereParts, expressionUsesTime);
return SqlExpression.and(
SqlExpression.or(
SqlExpression.and(...timeExpressions),
...compares.map(compare =>
SqlExpression.and(
...timeExpressions.map(timeExpression => shiftTimeInExpression(timeExpression, compare)),
),
),
),
...timelessExpressions,
);
}
function expressionUsesTime(expression: SqlExpression): boolean {
return shiftTimeInExpression(expression, 'P1D') !== expression;
}
export function shiftTimeInExpression(expression: SqlExpression, compare: string): SqlExpression {
return expression.walk(ex => {
if (ex instanceof SqlLiteral) {
// Works with: __time < TIMESTAMP '2022-01-02 03:04:05'
if (ex.isDate()) {
return F('TIME_SHIFT', ex, period, -1);
return F.timeShift(ex, compare, -1);
}
} else if (ex instanceof SqlFunction) {
const effectiveFunctionName = ex.getEffectiveFunctionName();
// Works with: TIME_IN_INTERVAL(__time, '<interval>')
if (effectiveFunctionName === 'TIME_IN_INTERVAL') {
return ex.changeArgs(ex.args!.change(0, F('TIME_SHIFT', ex.getArg(0), period, 1)));
// Ideally we could rewrite it to TIME_IN_INTERVAL(TIME_SHIFT(__time, period, 1), '<interval>') but that would be slow in the current Druid
// return ex.changeArgs(ex.args!.change(0, F('TIME_SHIFT', ex.getArg(0), period, 1)));a
const interval = ex.getArgAsString(1);
if (!interval) return ex;
const [start, end] = interval.split('/');
if (!IS_DATE_LIKE.test(start) || !IS_DATE_LIKE.test(end)) return ex;
const t = ex.getArg(0);
if (!t) return ex;
return F.timeShift(isoStringToTimestampLiteral(start), compare, -1)
.lessThanOrEqual(t)
.and(t.lessThan(F.timeShift(isoStringToTimestampLiteral(end), compare, -1)));
}
// Works with: TIME_SHIFT(...) <= __time
// and: __time < MAX_DATA_TIME()
if (effectiveFunctionName === 'TIME_SHIFT' || effectiveFunctionName === 'MAX_DATA_TIME') {
return F('TIME_SHIFT', ex, period, -1);
return F.timeShift(ex, compare, -1);
}
}

2
web-console/src/views/explore-view/utils/misc.ts
View File

@ -34,7 +34,7 @@ export function toggle<T>(xs: readonly T[], x: T, eq?: (a: T, b: T) => boolean):
}
export function getInitQuery(table: SqlExpression, where: SqlExpression): SqlQuery {
return SqlQuery.from(table).applyIf(String(where) !== 'TRUE', q =>
return SqlQuery.from(table.as('t')).applyIf(String(where) !== 'TRUE', q =>
q.changeWhereExpression(where),
);
}

14
web-console/src/views/lookups-view/lookups-view.tsx
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
import { Button, Icon, Intent } from '@blueprintjs/core';
import { Button, Icon, Intent, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import type { Filter } from 'react-table';
@ -295,8 +295,16 @@ export class LookupsView extends React.PureComponent<LookupsViewProps, LookupsVi
);
}}
confirmButtonText="Delete lookup"
successText="Lookup was deleted"
failText="Could not delete lookup"
successText={
<>
Lookup <Tag minimal>{deleteLookupName}</Tag> was deleted
</>
}
failText={
<>
Could not delete lookup <Tag minimal>{deleteLookupName}</Tag>
</>
}
intent={Intent.DANGER}
onClose={() => {
this.setState({ deleteLookupTier: undefined, deleteLookupName: undefined });

14
web-console/src/views/services-view/services-view.tsx
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
import { Button, ButtonGroup, Intent, Label, MenuItem } from '@blueprintjs/core';
import { Button, ButtonGroup, Intent, Label, MenuItem, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { sum } from 'd3-array';
import React from 'react';
@ -699,8 +699,16 @@ ORDER BY
return resp.data;
}}
confirmButtonText="Disable worker"
successText="Worker has been disabled"
failText="Could not disable worker"
successText={
<>
Worker <Tag minimal>{middleManagerDisableWorkerHost}</Tag> has been disabled
</>
}
failText={
<>
Could not disable worker <Tag minimal>{middleManagerDisableWorkerHost}</Tag>
</>
}
intent={Intent.DANGER}
onClose={() => {
this.setState({ middleManagerDisableWorkerHost: undefined });

2
web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
View File

@ -103,7 +103,7 @@ export const PreviewTable = React.memo(function PreviewTable(props: PreviewTable
);
}
const numericColumnBraces = getNumericColumnBraces(queryResult);
const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, undefined);
return (
<div className="preview-table">
<ReactTable

2
web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
View File

@ -515,7 +515,7 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
throw new DruidError(e);
}
return result.attachQuery({}, SqlQuery.maybeParse(previewQueryString));
return result.attachQuery({} as any, SqlQuery.maybeParse(previewQueryString));
}
},
backgroundStatusCheck: executionBackgroundResultStatusCheck,

2
web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
View File

@ -190,7 +190,7 @@ export class FlexibleQueryInput extends React.PureComponent<
const found = dedupe(findAllSqlQueriesInText(queryString), ({ startRowColumn }) =>
String(startRowColumn.row),
);
if (found.length <= 1) return []; // Do not highlight a single query or no queries
if (!found.length) return [];
// Do not report the first query if it is basically the main query minus whitespace
const firstQuery = found[0].sql;

2
web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
View File

@ -546,7 +546,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result
? parsedQuery.getSelectExpressionForIndex(editingColumn)
: undefined;
const numericColumnBraces = getNumericColumnBraces(queryResult, pagination);
const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, pagination);
return (
<div className={classNames('result-table-pane', { 'more-results': hasMoreResults })}>
{finalPage ? (