Web console: expose forceSegmentSortByTime (#16967)

* no force time

* time UI

* update menus

* tweaks

* dont use bp5

* nicer values

* update snapshots

* similar engine lables

* update snaps
This commit is contained in:
Vadim Ogievetsky 2024-08-29 09:58:15 -07:00 committed by GitHub
parent 1d292c5a59
commit 358d06abc1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 575 additions and 125 deletions

View File

@ -87,7 +87,7 @@ It is equivalent to the **Task** view in the **Ingestion** view with the filter
9. The **Preview** button appears when you enter an INSERT/REPLACE query. It runs the query inline without the INSERT/REPLACE clause and with an added LIMIT to give you a preview of the data that would be ingested if you click **Run**.
The added LIMIT makes the query run faster but provides incomplete results.
10. The engine selector lets you choose which engine (API endpoint) to send a query to. By default, it automatically picks which endpoint to use based on an analysis of the query, but you can select a specific engine explicitly. You can also configure the engine specific context parameters from this menu.
11. The **Max tasks** picker appears when you have the **sql-msq-task** engine selected. It lets you configure the degree of parallelism.
11. The **Max tasks** picker appears when you have the **SQL MSQ-task** engine selected. It lets you configure the degree of parallelism.
12. The More menu (**...**) contains the following helpful tools:
- **Explain SQL query** shows you the logical plan returned by `EXPLAIN PLAN FOR` for a SQL query.
- **Query history** shows you previously executed queries.

View File

@ -100,9 +100,9 @@ In this section you run some queries using aggregate functions and perform some
![aggregate-query](../assets/tutorial-sql-aggregate-query.png)
7. Click **Engine: auto (sql-native)** to display the engine options—**native** for native (JSON-based) queries, **sql-native** for Druid SQL queries, and **sql-msq-task** for SQL-based ingestion.
7. Click **Engine: Auto (SQL native)** to display the engine options—**Native** for native (JSON-based) queries, **SQL native** for Druid SQL queries, and **SQL MSQ-task** for SQL-based ingestion.
Select **auto** to let Druid select the most efficient engine based on your query input.
Select **Auto** to let Druid select the most efficient engine based on your query input.
8. From the engine menu you can also edit the query context and turn off some query defaults.

View File

@ -37,8 +37,8 @@ export * from './json-collapse/json-collapse';
export * from './json-input/json-input';
export * from './learn-more/learn-more';
export * from './loader/loader';
export * from './menu-boolean/menu-boolean';
export * from './menu-checkbox/menu-checkbox';
export * from './menu-tristate/menu-tristate';
export * from './more-button/more-button';
export * from './plural-pair-if-needed/plural-pair-if-needed';
export * from './popover-text/popover-text';

View File

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`MenuTristate matches snapshot false 1`] = `
exports[`MenuBoolean matches snapshot false 1`] = `
<li
class="bp5-submenu"
role="none"
@ -54,7 +54,7 @@ exports[`MenuTristate matches snapshot false 1`] = `
</li>
`;
exports[`MenuTristate matches snapshot undefined 1`] = `
exports[`MenuBoolean matches snapshot no undefined 1`] = `
<li
class="bp5-submenu"
role="none"
@ -68,7 +68,7 @@ exports[`MenuTristate matches snapshot undefined 1`] = `
aria-expanded="false"
aria-haspopup="menu"
class="bp5-menu-item menu-tristate"
label="auto (true)"
label="false"
role="none"
tabindex="-1"
>
@ -80,7 +80,61 @@ exports[`MenuTristate matches snapshot undefined 1`] = `
<span
class="bp5-menu-item-label"
>
auto (true)
false
</span>
<span
class="bp5-icon bp5-icon-caret-right bp5-submenu-icon"
>
<svg
data-icon="caret-right"
height="16"
role="img"
viewBox="0 0 16 16"
width="16"
>
<title>
Open sub menu
</title>
<path
d="M220 160C220 163 218.6 165.6 216.6 167.4L216.6 167.4L136.6 237.4L136.6 237.4C134.8 239 132.6 240 130 240C124.4 240 120 235.6 120 230V90C120 84.4 124.4 80 130 80C132.6 80 134.8 81 136.6 82.6C136.6 82.6 136.6 82.6 136.6 82.6L216.6 152.6L216.6 152.6C218.6 154.4 220 157 220 160z"
fill-rule="evenodd"
style="transform-origin: center;"
transform="scale(0.05, -0.05) translate(-160, -160)"
/>
</svg>
</span>
</a>
</span>
</li>
`;
exports[`MenuBoolean matches snapshot undefined 1`] = `
<li
class="bp5-submenu"
role="none"
>
<span
class="bp5-popover-target"
role="menuitem"
tabindex="0"
>
<a
aria-expanded="false"
aria-haspopup="menu"
class="bp5-menu-item menu-tristate"
label="Auto (true)"
role="none"
tabindex="-1"
>
<div
class="bp5-text-overflow-ellipsis bp5-fill"
>
hello
</div>
<span
class="bp5-menu-item-label"
>
Auto (true)
</span>
<span
class="bp5-icon bp5-icon-caret-right bp5-submenu-icon"

View File

@ -19,14 +19,15 @@
import { render } from '@testing-library/react';
import React from 'react';
import { MenuTristate } from './menu-tristate';
import { MenuBoolean } from './menu-boolean';
describe('MenuTristate', () => {
describe('MenuBoolean', () => {
it('matches snapshot undefined', () => {
const menuCheckbox = (
<MenuTristate
<MenuBoolean
text="hello"
value={undefined}
showUndefined
undefinedEffectiveValue
onValueChange={() => {}}
/>
@ -37,9 +38,10 @@ describe('MenuTristate', () => {
it('matches snapshot false', () => {
const menuCheckbox = (
<MenuTristate
<MenuBoolean
text="hello"
value={false}
showUndefined
undefinedEffectiveValue={false}
onValueChange={() => {}}
/>
@ -47,4 +49,10 @@ describe('MenuTristate', () => {
const { container } = render(menuCheckbox);
expect(container.firstChild).toMatchSnapshot();
});
it('matches snapshot no undefined', () => {
const menuCheckbox = <MenuBoolean text="hello" value={false} onValueChange={() => {}} />;
const { container } = render(menuCheckbox);
expect(container.firstChild).toMatchSnapshot();
});
});

View File

@ -19,50 +19,71 @@
import type { MenuItemProps } from '@blueprintjs/core';
import { MenuItem } from '@blueprintjs/core';
import classNames from 'classnames';
import type { ReactNode } from 'react';
import React from 'react';
import { tickIcon } from '../../utils';
export interface MenuTristateProps extends Omit<MenuItemProps, 'label'> {
value: boolean | undefined;
onValueChange(value: boolean | undefined): void;
undefinedLabel?: string;
undefinedEffectiveValue?: boolean;
export type TrueFalseUndefined = 'true' | 'false' | 'undefined';
function toKey(value: boolean | undefined) {
return String(value) as TrueFalseUndefined;
}
export function MenuTristate(props: MenuTristateProps) {
const DEFAULT_OPTIONS_TEXT: Partial<Record<TrueFalseUndefined, string>> = { undefined: 'Auto' };
export const ENABLE_DISABLE_OPTIONS_TEXT: Partial<Record<TrueFalseUndefined, string>> = {
true: 'Enable',
false: 'Disable',
undefined: 'Auto',
};
export interface MenuBooleanProps extends Omit<MenuItemProps, 'label'> {
value: boolean | undefined;
onValueChange(value: boolean | undefined): void;
showUndefined?: boolean;
undefinedEffectiveValue?: boolean;
optionsText?: Partial<Record<TrueFalseUndefined, string>>;
optionsLabelElement?: Partial<Record<TrueFalseUndefined, ReactNode>>;
}
export function MenuBoolean(props: MenuBooleanProps) {
const {
value,
onValueChange,
undefinedLabel,
showUndefined,
undefinedEffectiveValue,
className,
shouldDismissPopover,
optionsText = DEFAULT_OPTIONS_TEXT,
optionsLabelElement = {},
...rest
} = props;
const effectiveValue = showUndefined ? value : value ?? undefinedEffectiveValue;
const shouldDismiss = shouldDismissPopover ?? false;
function formatValue(value: boolean | undefined): string {
return String(value ?? undefinedLabel ?? 'auto');
const s = toKey(value);
return optionsText[s] ?? s;
}
return (
<MenuItem
className={classNames('menu-tristate', className)}
shouldDismissPopover={shouldDismiss}
label={
formatValue(value) +
(typeof value === 'undefined' && typeof undefinedEffectiveValue === 'boolean'
? ` (${undefinedEffectiveValue})`
: '')
}
label={`${formatValue(effectiveValue)}${
typeof effectiveValue === 'undefined' && typeof undefinedEffectiveValue === 'boolean'
? ` (${formatValue(undefinedEffectiveValue)})`
: ''
}`}
{...rest}
>
{[undefined, true, false].map((v, i) => (
{(showUndefined ? [undefined, true, false] : [true, false]).map(v => (
<MenuItem
key={i}
icon={tickIcon(value === v)}
key={String(v)}
icon={tickIcon(effectiveValue === v)}
text={formatValue(v)}
labelElement={optionsLabelElement[toKey(v)]}
onClick={() => onValueChange(v)}
shouldDismissPopover={shouldDismiss}
/>

View File

@ -18,9 +18,10 @@
import type { Field } from '../../components';
import { filterMap, typeIsKnown } from '../../utils';
import type { SampleResponse } from '../../utils/sampler';
import type { SampleResponse, TimeColumnAction } from '../../utils/sampler';
import { getHeaderNamesFromSampleResponse } from '../../utils/sampler';
import { guessColumnTypeFromSampleResponse } from '../ingestion-spec/ingestion-spec';
import { TIME_COLUMN } from '../timestamp-spec/timestamp-spec';
export interface DimensionsSpec {
readonly dimensions?: (string | DimensionSpec)[];
@ -28,6 +29,7 @@ export interface DimensionsSpec {
readonly spatialDimensions?: any[];
readonly includeAllDimensions?: boolean;
readonly useSchemaDiscovery?: boolean;
readonly forceSegmentSortByTime?: boolean;
}
export interface DimensionSpec {
@ -61,6 +63,7 @@ export const DIMENSION_SPEC_FIELDS: Field<DimensionSpec>[] = [
type: 'string',
required: true,
suggestions: KNOWN_TYPES,
disabled: d => d.name === TIME_COLUMN,
},
{
name: 'createBitmapIndex',
@ -163,8 +166,13 @@ export function getDimensionSpecs(
guessNumericStringsAsNumbers: boolean,
forceMvdInsteadOfArray: boolean,
hasRollup: boolean,
timeColumnAction: TimeColumnAction,
): (string | DimensionSpec)[] {
return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, 'ignore'), h => {
return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, timeColumnAction), h => {
if (h === TIME_COLUMN) {
return { type: 'long', name: h };
}
const columnTypeHint = columnTypeHints[h];
const guessedColumnType = guessColumnTypeFromSampleResponse(
sampleResponse,

View File

@ -22,6 +22,7 @@ import type { IngestionSpec } from './ingestion-spec';
import {
adjustId,
cleanSpec,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
guessColumnTypeFromInput,
guessColumnTypeFromSampleResponse,
guessKafkaInputFormat,
@ -857,10 +858,94 @@ describe('spec utils', () => {
});
describe('updateSchemaWithSample', () => {
it('works with when not forcing time, arrays', () => {
const updateSpec = updateSchemaWithSample(
ingestionSpec,
JSON_SAMPLE,
false,
'fixed',
'arrays',
true,
);
expect(updateSpec.spec).toMatchInlineSnapshot(`
{
"dataSchema": {
"dataSource": "wikipedia",
"dimensionsSpec": {
"dimensions": [
{
"name": "__time",
"type": "long",
},
"user",
"id",
{
"castToType": "ARRAY<STRING>",
"name": "tags",
"type": "auto",
},
{
"castToType": "ARRAY<LONG>",
"name": "nums",
"type": "auto",
},
],
"forceSegmentSortByTime": false,
},
"granularitySpec": {
"queryGranularity": "hour",
"rollup": true,
"segmentGranularity": "day",
},
"metricsSpec": [
{
"name": "count",
"type": "count",
},
{
"fieldName": "followers",
"name": "sum_followers",
"type": "longSum",
},
{
"fieldName": "spend",
"name": "sum_spend",
"type": "doubleSum",
},
],
"timestampSpec": {
"column": "timestamp",
"format": "iso",
},
},
"ioConfig": {
"inputFormat": {
"type": "json",
},
"inputSource": {
"type": "http",
"uris": [
"https://website.com/wikipedia.json.gz",
],
},
"type": "index_parallel",
},
"tuningConfig": {
"forceGuaranteedRollup": true,
"partitionsSpec": {
"type": "hashed",
},
"type": "index_parallel",
},
}
`);
});
it('works with rollup, arrays', () => {
const updateSpec = updateSchemaWithSample(
ingestionSpec,
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
'arrays',
true,
@ -938,6 +1023,7 @@ describe('spec utils', () => {
const updateSpec = updateSchemaWithSample(
ingestionSpec,
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
'multi-values',
true,
@ -1015,6 +1101,7 @@ describe('spec utils', () => {
const updatedSpec = updateSchemaWithSample(
ingestionSpec,
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
'arrays',
false,
@ -1083,6 +1170,7 @@ describe('spec utils', () => {
const updatedSpec = updateSchemaWithSample(
ingestionSpec,
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
'multi-values',
false,

View File

@ -295,6 +295,13 @@ export type SchemaMode = 'fixed' | 'string-only-discovery' | 'type-aware-discove
export type ArrayMode = 'arrays' | 'multi-values';
export const DEFAULT_FORCE_SEGMENT_SORT_BY_TIME = true;
export function getForceSegmentSortByTime(spec: Partial<IngestionSpec>): boolean {
return (
deepGet(spec, 'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime') ??
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME
);
}
export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
if (deepGet(spec, 'spec.dataSchema.dimensionsSpec.useSchemaDiscovery') === true) {
return 'type-aware-discovery';
@ -2744,6 +2751,7 @@ function getColumnTypeHintsFromSpec(spec: Partial<IngestionSpec>): Record<string
export function updateSchemaWithSample(
spec: Partial<IngestionSpec>,
sampleResponse: SampleResponse,
forceSegmentSortByTime: boolean,
schemaMode: SchemaMode,
arrayMode: ArrayMode,
rollup: boolean,
@ -2756,6 +2764,15 @@ export function updateSchemaWithSample(
let newSpec = spec;
newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime');
if (forceSegmentSortByTime !== DEFAULT_FORCE_SEGMENT_SORT_BY_TIME) {
newSpec = deepSet(
newSpec,
'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime',
forceSegmentSortByTime,
);
}
switch (schemaMode) {
case 'type-aware-discovery':
newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.useSchemaDiscovery', true);
@ -2784,6 +2801,7 @@ export function updateSchemaWithSample(
guessNumericStringsAsNumbers,
arrayMode === 'multi-values',
rollup,
forceSegmentSortByTime ?? DEFAULT_FORCE_SEGMENT_SORT_BY_TIME ? 'ignore' : 'preserve',
),
);
break;

View File

@ -41,6 +41,7 @@ export interface QueryContext {
failOnEmptyInsert?: boolean;
waitUntilSegmentsLoad?: boolean;
useConcurrentLocks?: boolean;
forceSegmentSortByTime?: boolean;
[key: string]: any;
}
@ -63,6 +64,7 @@ export const DEFAULT_SERVER_QUERY_CONTEXT: QueryContext = {
failOnEmptyInsert: false,
waitUntilSegmentsLoad: false,
useConcurrentLocks: false,
forceSegmentSortByTime: true,
};
export interface QueryWithContext {

View File

@ -63,3 +63,9 @@ body {
width: 100%;
}
}
// Prevent popover menus from being longer than 45% of available height, let them scroll instead
.#{$bp-ns}-popover-content > .#{$bp-ns}-menu {
max-height: 47vh;
overflow: auto;
}

View File

@ -86,6 +86,14 @@ export function convertSpecToSql(spec: any): QueryWithContext {
context.arrayIngestMode = 'array';
}
const forceSegmentSortByTime = deepGet(
spec,
'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime',
);
if (typeof forceSegmentSortByTime !== 'undefined') {
context.forceSegmentSortByTime = forceSegmentSortByTime;
}
const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
if (indexSpec) {
context.indexSpec = indexSpec;

View File

@ -73,16 +73,18 @@ export interface SampleResponse {
numRowsRead: number;
}
export type TimeColumnAction = 'preserve' | 'ignore' | 'ignoreIfZero';
export function getHeaderNamesFromSampleResponse(
sampleResponse: SampleResponse,
timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
timeColumnAction: TimeColumnAction = 'preserve',
): string[] {
return getHeaderFromSampleResponse(sampleResponse, timeColumnAction).map(s => s.name);
}
export function getHeaderFromSampleResponse(
sampleResponse: SampleResponse,
timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
timeColumnAction: TimeColumnAction = 'preserve',
): { name: string; type: string }[] {
const ignoreTimeColumn =
timeColumnAction === 'ignore' ||
@ -462,13 +464,17 @@ export async function sampleForTimestamp(
export async function sampleForTransform(
spec: Partial<IngestionSpec>,
cacheRows: CacheRows,
forceSegmentSortByTime: boolean,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec');
const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || [];
// Extra step to simulate auto-detecting dimension with transforms
let specialDimensionSpec: DimensionsSpec = { useSchemaDiscovery: true };
let specialDimensionSpec: DimensionsSpec = {
useSchemaDiscovery: true,
forceSegmentSortByTime,
};
if (transforms && transforms.length) {
const sampleSpecHack: SampleSpec = {
type: samplerType,

View File

@ -83,6 +83,7 @@ import {
computeFlattenPathsForData,
CONSTANT_TIMESTAMP_SPEC,
CONSTANT_TIMESTAMP_SPEC_FIELDS,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
DIMENSION_SPEC_FIELDS,
fillDataSourceNameIfNeeded,
fillInputFormatIfNeeded,
@ -92,6 +93,7 @@ import {
getArrayMode,
getDimensionSpecName,
getFlattenSpec,
getForceSegmentSortByTime,
getIngestionComboType,
getIngestionImage,
getIngestionTitle,
@ -311,7 +313,14 @@ function initializeSchemaWithSampleIfNeeded(
sample: SampleResponse,
): Partial<IngestionSpec> {
if (deepGet(spec, 'spec.dataSchema.dimensionsSpec')) return spec;
return updateSchemaWithSample(spec, sample, 'fixed', 'multi-values', getRollup(spec, false));
return updateSchemaWithSample(
spec,
sample,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
'multi-values',
getRollup(spec, false),
);
}
type Step =
@ -394,6 +403,7 @@ export interface LoadDataViewState {
continueToSpec: boolean;
showResetConfirm: boolean;
newRollup?: boolean;
newForceSegmentSortByTime?: boolean;
newSchemaMode?: SchemaMode;
newArrayMode?: ArrayMode;
@ -1965,7 +1975,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
let sampleResponse: SampleResponse;
try {
sampleResponse = await sampleForTransform(spec, cacheRows);
sampleResponse = await sampleForTransform(
spec,
cacheRows,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
);
} catch (e) {
this.setState(({ transformQueryState }) => ({
transformQueryState: new QueryState({
@ -2359,6 +2373,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const somethingSelected = Boolean(
selectedAutoDimension || selectedDimensionSpec || selectedMetricSpec,
);
const forceSegmentSortByTime = getForceSegmentSortByTime(spec);
const schemaMode = getSchemaMode(spec);
const arrayMode = getArrayMode(spec);
@ -2404,6 +2419,39 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<SchemaMessage schemaMode={schemaMode} />
{!somethingSelected && (
<>
<FormGroupWithInfo
inlineInfo
info={
<PopoverText>
<p>
When set to true (the default), segments created by the ingestion job are
sorted by <Code>{'{__time, dimensions[0], dimensions[1], ...}'}</Code>. When
set to false, segments created by the ingestion job are sorted by{' '}
<Code>{'{dimensions[0], dimensions[1], ...}'}</Code>. To include{' '}
<Code>__time</Code> in the sort order when this parameter is set to{' '}
<Code>false</Code>, you must include a dimension named <Code>__time</Code>{' '}
with type <Code>long</Code> explicitly in the `dimensions` list.
</p>
<p>
Setting this to `false` is an experimental feature; see
<ExternalLink href={`${getLink('DOCS')}/ingestion/partitioning/#sorting`}>
Sorting
</ExternalLink>{' '}
for details.
</p>
</PopoverText>
}
>
<Switch
checked={forceSegmentSortByTime}
onChange={() =>
this.setState({
newForceSegmentSortByTime: !forceSegmentSortByTime,
})
}
label="Force segment sort by time"
/>
</FormGroupWithInfo>
<FormGroupWithInfo
inlineInfo
info={
@ -2563,7 +2611,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
{schemaToolsMenu && (
<FormGroup>
<Popover content={schemaToolsMenu}>
<Button icon={IconNames.BUILD} />
<Button icon={IconNames.BUILD} text="Tools" />
</Popover>
</FormGroup>
)}
@ -2572,6 +2620,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
{this.renderAutoDimensionControls()}
{this.renderDimensionSpecControls()}
{this.renderMetricSpecControls()}
{this.renderChangeForceSegmentSortByTime()}
{this.renderChangeRollupAction()}
{this.renderChangeSchemaModeAction()}
{this.renderChangeArrayModeAction()}
@ -2660,6 +2709,44 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
});
};
renderChangeForceSegmentSortByTime() {
const { newForceSegmentSortByTime, spec, cacheRows } = this.state;
if (typeof newForceSegmentSortByTime === 'undefined' || !cacheRows) return;
return (
<AsyncActionDialog
action={async () => {
const sampleResponse = await sampleForTransform(
spec,
cacheRows,
newForceSegmentSortByTime,
);
this.updateSpec(
updateSchemaWithSample(
spec,
sampleResponse,
newForceSegmentSortByTime,
getSchemaMode(spec),
getArrayMode(spec),
getRollup(spec),
true,
),
);
}}
confirmButtonText={`Yes - ${
newForceSegmentSortByTime ? 'force time to be first' : "don't force time to be first"
}`}
successText={`forceSegmentSortByTime was set to ${newForceSegmentSortByTime}.`}
failText="Could change rollup"
intent={Intent.WARNING}
onClose={() => this.setState({ newForceSegmentSortByTime: undefined })}
>
<p>{`Are you sure you want to set forceSegmentSortByTime to ${newForceSegmentSortByTime}?`}</p>
<p>Making this change will reset any work you have done in this section.</p>
</AsyncActionDialog>
);
}
renderChangeRollupAction() {
const { newRollup, spec, cacheRows } = this.state;
if (typeof newRollup === 'undefined' || !cacheRows) return;
@ -2667,11 +2754,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
return (
<AsyncActionDialog
action={async () => {
const sampleResponse = await sampleForTransform(spec, cacheRows);
const sampleResponse = await sampleForTransform(
spec,
cacheRows,
getForceSegmentSortByTime(spec),
);
this.updateSpec(
updateSchemaWithSample(
spec,
sampleResponse,
getForceSegmentSortByTime(spec),
getSchemaMode(spec),
getArrayMode(spec),
newRollup,
@ -2699,11 +2791,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
return (
<AsyncActionDialog
action={async () => {
const sampleResponse = await sampleForTransform(spec, cacheRows);
const sampleResponse = await sampleForTransform(
spec,
cacheRows,
getForceSegmentSortByTime(spec),
);
this.updateSpec(
updateSchemaWithSample(
spec,
sampleResponse,
getForceSegmentSortByTime(spec),
newSchemaMode,
getArrayMode(spec),
getRollup(spec),
@ -2766,11 +2863,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
return (
<AsyncActionDialog
action={async () => {
const sampleResponse = await sampleForTransform(spec, cacheRows);
const sampleResponse = await sampleForTransform(
spec,
cacheRows,
getForceSegmentSortByTime(spec),
);
this.updateSpec(
updateSchemaWithSample(
spec,
sampleResponse,
getForceSegmentSortByTime(spec),
getSchemaMode(spec),
newArrayMode,
getRollup(spec),
@ -2829,6 +2931,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
renderDimensionSpecControls() {
const { spec, selectedDimensionSpec } = this.state;
if (!selectedDimensionSpec) return;
const selectedTime = selectedDimensionSpec.value.name === TIME_COLUMN;
const schemaMode = getSchemaMode(spec);
const dimensions = deepGet(spec, `spec.dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
@ -2913,7 +3016,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
)
}
showDelete={selectedDimensionSpec.index !== -1}
disableDelete={dimensions.length <= 1}
disableDelete={dimensions.length <= 1 || selectedTime}
onDelete={() =>
this.updateSpec(
deepDelete(
@ -2934,7 +3037,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</Popover>
</FormGroup>
)}
{selectedDimensionSpec.index !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
{selectedDimensionSpec.index !== -1 &&
deepGet(spec, 'spec.dataSchema.metricsSpec') &&
!selectedTime && (
<FormGroup>
<Popover content={convertToMetricMenu}>
<Button

View File

@ -137,11 +137,9 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
<div
className="clickable"
onClick={() => {
if (isTimestamp) return;
if (definedDimensions && dimensionSpec) {
onDimensionSelect(inflateDimensionSpec(dimensionSpec), dimensionSpecIndex);
} else {
} else if (!isTimestamp) {
onAutoDimensionSelect(columnName);
}
}}

View File

@ -22,6 +22,7 @@ import {
ButtonGroup,
Callout,
FormGroup,
Icon,
Intent,
Menu,
MenuDivider,
@ -45,7 +46,13 @@ import { select, selectAll } from 'd3-selection';
import type { JSX } from 'react';
import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
import { ClearableInput, LearnMore, Loader } from '../../../components';
import {
ClearableInput,
ENABLE_DISABLE_OPTIONS_TEXT,
LearnMore,
Loader,
MenuBoolean,
} from '../../../components';
import { AsyncActionDialog } from '../../../dialogs';
import type { Execution, ExternalConfig, IngestQueryPattern } from '../../../druid-models';
import {
@ -100,6 +107,8 @@ import { RollupAnalysisPane } from './rollup-analysis-pane/rollup-analysis-pane'
import './schema-step.scss';
const EXPERIMENTAL_ICON = <Icon icon={IconNames.WARNING_SIGN} title="Experimental" />;
const queryRunner = new QueryRunner();
function digestQueryString(queryString: string): {
@ -248,6 +257,8 @@ interface EditorColumn {
export interface SchemaStepProps {
queryString: string;
onQueryStringChange(queryString: string): void;
forceSegmentSortByTime: boolean;
changeForceSegmentSortByTime(forceSegmentSortByTime: boolean): void;
enableAnalyze: boolean;
goToQuery: () => void;
onBack(): void;
@ -259,6 +270,8 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
const {
queryString,
onQueryStringChange,
forceSegmentSortByTime,
changeForceSegmentSortByTime,
enableAnalyze,
goToQuery,
onBack,
@ -640,6 +653,7 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
<MenuItem
icon={IconNames.CROSS}
text="Remove"
shouldDismissPopover={false}
onClick={() =>
updatePattern({
...ingestQueryPattern,
@ -652,17 +666,13 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
<MenuItem icon={IconNames.PLUS} text="Add column clustering">
{filterMap(ingestQueryPattern.dimensions, (dimension, i) => {
const outputName = dimension.getOutputName();
if (
outputName === TIME_COLUMN ||
ingestQueryPattern.clusteredBy.includes(i)
) {
return;
}
if (ingestQueryPattern.clusteredBy.includes(i)) return;
return (
<MenuItem
key={i}
text={outputName}
disabled={outputName === TIME_COLUMN && forceSegmentSortByTime}
onClick={() =>
updatePattern({
...ingestQueryPattern,
@ -674,6 +684,15 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {
);
})}
</MenuItem>
<MenuDivider />
<MenuBoolean
icon={IconNames.GEOTIME}
text="Force segment sort by time"
value={forceSegmentSortByTime}
onValueChange={v => changeForceSegmentSortByTime(Boolean(v))}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
/>
</Menu>
}
>

View File

@ -33,6 +33,7 @@ import {
DEFAULT_SERVER_QUERY_CONTEXT,
Execution,
externalConfigToIngestQueryPattern,
getQueryContextKey,
ingestQueryPatternToQuery,
} from '../../druid-models';
import type { Capabilities } from '../../helpers';
@ -147,6 +148,17 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
<SchemaStep
queryString={content.queryString}
onQueryStringChange={queryString => setContent({ ...content, queryString })}
forceSegmentSortByTime={getQueryContextKey(
'forceSegmentSortByTime',
content.queryContext || {},
serverQueryContext,
)}
changeForceSegmentSortByTime={forceSegmentSortByTime =>
setContent({
...content,
queryContext: { ...content?.queryContext, forceSegmentSortByTime },
})
}
enableAnalyze={false}
goToQuery={() => goToQuery(content)}
onBack={() => setContent(undefined)}

View File

@ -80,10 +80,17 @@ exports[`MaxTasksButton matches snapshot 1`] = `
active={false}
disabled={false}
icon="flow-branch"
label="max"
label="Max"
multiline={false}
popoverProps={{}}
shouldDismissPopover={true}
submenuProps={
{
"style": {
"width": 300,
},
}
}
text="Task assignment"
>
<Blueprint5.MenuItem
@ -97,10 +104,10 @@ exports[`MaxTasksButton matches snapshot 1`] = `
text={
<React.Fragment>
<strong>
max
Max
</strong>
:
Use as many tasks as possible, up to the maximum.
uses the maximum possible tasks up to the specified limit.
</React.Fragment>
}
/>
@ -108,6 +115,13 @@ exports[`MaxTasksButton matches snapshot 1`] = `
active={false}
disabled={false}
icon="blank"
labelElement={
<Blueprint5.Button
icon="help"
minimal={true}
onClick={[Function]}
/>
}
multiline={true}
onClick={[Function]}
popoverProps={{}}
@ -115,10 +129,10 @@ exports[`MaxTasksButton matches snapshot 1`] = `
text={
<React.Fragment>
<strong>
auto
Auto
</strong>
:
Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.
maximizes the number of tasks while staying within 512 MiB or 10,000 files per task, unless more tasks are needed to stay under the max task limit.
</React.Fragment>
}
/>

View File

@ -19,20 +19,33 @@
import type { ButtonProps } from '@blueprintjs/core';
import { Button, Menu, MenuDivider, MenuItem, Popover, Position } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { JSX } from 'react';
import type { JSX, ReactNode } from 'react';
import React, { useState } from 'react';
import { NumericInputDialog } from '../../../dialogs';
import type { QueryContext, TaskAssignment } from '../../../druid-models';
import { getQueryContextKey } from '../../../druid-models';
import { deleteKeys, formatInteger, tickIcon } from '../../../utils';
import { getLink } from '../../../links';
import { capitalizeFirst, deleteKeys, formatInteger, tickIcon } from '../../../utils';
const MAX_NUM_TASK_OPTIONS = [2, 3, 4, 5, 7, 9, 11, 17, 33, 65, 129];
const TASK_ASSIGNMENT_OPTIONS: TaskAssignment[] = ['max', 'auto'];
const TASK_ASSIGNMENT_DESCRIPTION: Record<string, string> = {
max: 'Use as many tasks as possible, up to the maximum.',
auto: `Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.`,
max: 'uses the maximum possible tasks up to the specified limit.',
auto: 'maximizes the number of tasks while staying within 512 MiB or 10,000 files per task, unless more tasks are needed to stay under the max task limit.',
};
const TASK_ASSIGNMENT_LABEL_ELEMENT: Record<string, ReactNode> = {
auto: (
<Button
icon={IconNames.HELP}
minimal
onClick={() =>
window.open(`${getLink('DOCS')}/multi-stage-query/reference#context-parameters`, '_blank')
}
/>
),
};
const DEFAULT_MAX_NUM_TASKS_LABEL_FN = (maxNum: number) => {
@ -113,16 +126,22 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps
onClick={() => setCustomMaxNumTasksDialogOpen(true)}
/>
<MenuDivider />
<MenuItem icon={IconNames.FLOW_BRANCH} text="Task assignment" label={taskAssigment}>
<MenuItem
icon={IconNames.FLOW_BRANCH}
text="Task assignment"
label={capitalizeFirst(taskAssigment)}
submenuProps={{ style: { width: 300 } }}
>
{TASK_ASSIGNMENT_OPTIONS.map(t => (
<MenuItem
key={String(t)}
icon={tickIcon(t === taskAssigment)}
text={
<>
<strong>{t}</strong>: {TASK_ASSIGNMENT_DESCRIPTION[t]}
<strong>{capitalizeFirst(t)}</strong>: {TASK_ASSIGNMENT_DESCRIPTION[t]}
</>
}
labelElement={TASK_ASSIGNMENT_LABEL_ELEMENT[t]}
shouldDismissPopover={false}
multiline
onClick={() => changeQueryContext({ ...queryContext, taskAssignment: t })}

View File

@ -46,7 +46,7 @@ exports[`RunPanel matches snapshot on msq (auto) query 1`] = `
<span
class="bp5-button-text"
>
Engine: sql-msq-task
Engine: SQL MSQ-task
</span>
<span
aria-hidden="true"
@ -150,7 +150,7 @@ exports[`RunPanel matches snapshot on native (auto) query 1`] = `
<span
class="bp5-button-text"
>
Engine: auto (sql-native)
Engine: Auto (SQL native)
</span>
<span
aria-hidden="true"

View File

@ -33,7 +33,7 @@ import { IconNames } from '@blueprintjs/icons';
import type { JSX } from 'react';
import React, { useCallback, useMemo, useState } from 'react';
import { MenuCheckbox, MenuTristate } from '../../../components';
import { ENABLE_DISABLE_OPTIONS_TEXT, MenuBoolean, MenuCheckbox } from '../../../components';
import { EditContextDialog, StringInputDialog } from '../../../dialogs';
import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
import type {
@ -76,28 +76,52 @@ const NAMED_TIMEZONES: string[] = [
'Australia/Sydney', // +11.0
];
const ARRAY_INGEST_MODE_LABEL: Record<ArrayIngestMode, string> = {
array: 'Array',
mvd: 'MVD',
};
const ARRAY_INGEST_MODE_DESCRIPTION: Record<ArrayIngestMode, JSX.Element> = {
array: (
<>
array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
Array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
<Tag minimal>ARRAY&lt;STRING&gt;</Tag>
</>
),
mvd: (
<>
mvd: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
MVD: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
</>
),
};
const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
if (!engine) return { text: 'auto' };
return {
text: engine,
label: engine === 'sql-msq-task' ? 'multi-stage-query' : undefined,
};
const SQL_JOIN_ALGORITHM_LABEL: Record<SqlJoinAlgorithm, string> = {
broadcast: 'Broadcast',
sortMerge: 'Sort merge',
};
const SELECT_DESTINATION_LABEL: Record<SelectDestination, string> = {
taskReport: 'Task report',
durableStorage: 'Durable storage',
};
const DEFAULT_ENGINES_LABEL_FN = (engine: DruidEngine | undefined) => {
switch (engine) {
case 'native':
return { text: 'Native' };
case 'sql-native':
return { text: 'SQL native' };
case 'sql-msq-task':
return { text: 'SQL MSQ-task', label: 'multi-stage-query' };
default:
return { text: 'Auto' };
}
};
const EXPERIMENTAL_ICON = <Icon icon={IconNames.WARNING_SIGN} title="Experimental" />;
export interface RunPanelProps
extends Pick<MaxTasksButtonProps, 'maxTasksLabelFn' | 'fullClusterCapacityLabelFn'> {
query: WorkbenchQuery;
@ -169,6 +193,11 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
queryContext,
defaultQueryContext,
);
const forceSegmentSortByTime = getQueryContextKey(
'forceSegmentSortByTime',
queryContext,
defaultQueryContext,
);
const finalizeAggregations = queryContext.finalizeAggregations;
const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
const groupByEnableMultiValueUnnesting = queryContext.groupByEnableMultiValueUnnesting;
@ -313,13 +342,13 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
)}
<MenuItem
icon={IconNames.PROPERTIES}
text="Edit context"
text="Edit query context..."
onClick={() => setEditContextDialogOpen(true)}
label={pluralIfNeeded(numContextKeys, 'key')}
/>
<MenuItem
icon={IconNames.HELP}
text="Define parameters"
text="Define parameters..."
onClick={() => setEditParametersDialogOpen(true)}
label={queryParameters ? pluralIfNeeded(queryParameters.length, 'parameter') : ''}
/>
@ -365,37 +394,52 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
{effectiveEngine === 'sql-msq-task' ? (
<>
<MenuItem icon={IconNames.BRING_DATA} text="INSERT / REPLACE specific context">
<MenuCheckbox
checked={useConcurrentLocks}
text="Use concurrent locks"
onChange={() =>
<MenuBoolean
text="Force segment sort by time"
value={forceSegmentSortByTime}
onValueChange={forceSegmentSortByTime =>
changeQueryContext({
...queryContext,
useConcurrentLocks: !useConcurrentLocks,
forceSegmentSortByTime,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
/>
<MenuTristate
icon={IconNames.DISABLE}
<MenuBoolean
text="Use concurrent locks"
value={useConcurrentLocks}
onValueChange={useConcurrentLocks =>
changeQueryContext({
...queryContext,
useConcurrentLocks,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
optionsLabelElement={{ true: EXPERIMENTAL_ICON }}
/>
<MenuBoolean
text="Fail on empty insert"
value={failOnEmptyInsert}
showUndefined
undefinedEffectiveValue={false}
onValueChange={failOnEmptyInsert =>
changeQueryContext({ ...queryContext, failOnEmptyInsert })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuTristate
icon={IconNames.STOPWATCH}
<MenuBoolean
text="Wait until segments have loaded"
value={waitUntilSegmentsLoad}
showUndefined
undefinedEffectiveValue={ingestMode}
onValueChange={waitUntilSegmentsLoad =>
changeQueryContext({ ...queryContext, waitUntilSegmentsLoad })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuItem
icon={IconNames.TH_DERIVED}
text="Edit index spec"
text="Edit index spec..."
label={summarizeIndexSpec(indexSpec)}
shouldDismissPopover={false}
onClick={() => {
@ -420,34 +464,41 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
/>
))}
</MenuItem>
<MenuTristate
<MenuBoolean
icon={IconNames.TRANSLATE}
text="Finalize aggregations"
value={finalizeAggregations}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={finalizeAggregations =>
changeQueryContext({ ...queryContext, finalizeAggregations })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuTristate
<MenuBoolean
icon={IconNames.FORK}
text="Enable GroupBy multi-value unnesting"
text="GROUP BY multi-value unnesting"
value={groupByEnableMultiValueUnnesting}
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={groupByEnableMultiValueUnnesting =>
changeQueryContext({ ...queryContext, groupByEnableMultiValueUnnesting })
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuItem
icon={IconNames.INNER_JOIN}
text="Join algorithm"
label={sqlJoinAlgorithm}
label={
SQL_JOIN_ALGORITHM_LABEL[sqlJoinAlgorithm as SqlJoinAlgorithm] ??
sqlJoinAlgorithm
}
>
{(['broadcast', 'sortMerge'] as SqlJoinAlgorithm[]).map(o => (
<MenuItem
key={o}
icon={tickIcon(sqlJoinAlgorithm === o)}
text={o}
text={SQL_JOIN_ALGORITHM_LABEL[o]}
shouldDismissPopover={false}
onClick={() =>
changeQueryContext({ ...queryContext, sqlJoinAlgorithm: o })
@ -458,14 +509,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
<MenuItem
icon={IconNames.MANUALLY_ENTERED_DATA}
text="SELECT destination"
label={selectDestination}
label={
SELECT_DESTINATION_LABEL[selectDestination as SelectDestination] ??
selectDestination
}
intent={intent}
>
{(['taskReport', 'durableStorage'] as SelectDestination[]).map(o => (
<MenuItem
key={o}
icon={tickIcon(selectDestination === o)}
text={o}
text={SELECT_DESTINATION_LABEL[o]}
shouldDismissPopover={false}
onClick={() =>
changeQueryContext({ ...queryContext, selectDestination: o })
@ -486,52 +540,60 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
}}
/>
</MenuItem>
<MenuCheckbox
checked={durableShuffleStorage}
<MenuBoolean
icon={IconNames.CLOUD_TICK}
text="Durable shuffle storage"
onChange={() =>
value={durableShuffleStorage}
onValueChange={durableShuffleStorage =>
changeQueryContext({
...queryContext,
durableShuffleStorage: !durableShuffleStorage,
durableShuffleStorage,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
</>
) : (
<>
<MenuCheckbox
checked={useCache}
<MenuBoolean
icon={IconNames.DATA_CONNECTION}
text="Use cache"
onChange={() =>
value={useCache}
onValueChange={useCache =>
changeQueryContext({
...queryContext,
useCache: !useCache,
populateCache: !useCache,
useCache,
populateCache: useCache,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
<MenuCheckbox
checked={useApproximateTopN}
text="Use approximate TopN"
onChange={() =>
<MenuBoolean
icon={IconNames.HORIZONTAL_BAR_CHART_DESC}
text="Approximate TopN"
value={useApproximateTopN}
onValueChange={useApproximateTopN =>
changeQueryContext({
...queryContext,
useApproximateTopN: !useApproximateTopN,
useApproximateTopN,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
</>
)}
{effectiveEngine !== 'native' && (
<MenuCheckbox
checked={useApproximateCountDistinct}
text="Use approximate COUNT(DISTINCT)"
onChange={() =>
<MenuBoolean
icon={IconNames.ROCKET_SLANT}
text="Approximate COUNT(DISTINCT)"
value={useApproximateCountDistinct}
onValueChange={useApproximateCountDistinct =>
changeQueryContext({
...queryContext,
useApproximateCountDistinct: !useApproximateCountDistinct,
useApproximateCountDistinct,
})
}
optionsText={ENABLE_DISABLE_OPTIONS_TEXT}
/>
)}
{effectiveEngine === 'sql-native' && (
@ -603,7 +665,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
}
>
<Button
text={`Array ingest mode: ${arrayIngestMode ?? '(server default)'}`}
text={`Array ingest mode: ${
arrayIngestMode ? ARRAY_INGEST_MODE_LABEL[arrayIngestMode] : '(server default)'
}`}
rightIcon={IconNames.CARET_DOWN}
/>
</Popover>