Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Web console: Make array ingest mode ux better (#15927) 

* only set arrayIngestMode: array when needed (still do the queries the correct way)

* arrayIngestMode control

* update wording

* feedback fixes
This commit is contained in:
Vadim Ogievetsky 2024-03-13 13:04:22 -07:00 committed by GitHub
parent 03c191f701
commit ccae19a546
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 107 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
View File

@ -297,7 +297,10 @@ export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
return Array.isArray(dimensions) && dimensions.length === 0 ? 'string-only-discovery' : 'fixed';
}
export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
export function getArrayMode(
spec: Partial<IngestionSpec>,
whenUnclear: ArrayMode = 'arrays',
): ArrayMode {
const schemaMode = getSchemaMode(spec);
switch (schemaMode) {
case 'type-aware-discovery':
@ -332,7 +335,7 @@ export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
return 'multi-values';
}
return 'arrays';
return whenUnclear;
}
}
}

21
web-console/src/druid-models/query-context/query-context.tsx
View File

@ -18,6 +18,8 @@
import { deepDelete, deepSet } from '../../utils';
export type ArrayIngestMode = 'array' | 'mvd';
export interface QueryContext {
useCache?: boolean;
populateCache?: boolean;
@ -32,7 +34,7 @@ export interface QueryContext {
durableShuffleStorage?: boolean;
maxParseExceptions?: number;
groupByEnableMultiValueUnnesting?: boolean;
arrayIngestMode?: 'array' | 'mvd';
arrayIngestMode?: ArrayIngestMode;
[key: string]: any;
}
@ -248,3 +250,20 @@ export function changeMaxParseExceptions(
return deepDelete(context, 'maxParseExceptions');
}
}
// arrayIngestMode
export function getArrayIngestMode(context: QueryContext): ArrayIngestMode | undefined {
return context.arrayIngestMode;
}
export function changeArrayIngestMode(
context: QueryContext,
arrayIngestMode: ArrayIngestMode | undefined,
): QueryContext {
if (arrayIngestMode) {
return deepSet(context, 'arrayIngestMode', arrayIngestMode);
} else {
return deepDelete(context, 'arrayIngestMode');
}
}

6
web-console/src/druid-models/workbench-query/workbench-query.ts
View File

@ -94,6 +94,8 @@ export class WorkbenchQuery {
partitionedByHint: string | undefined,
arrayMode: ArrayMode,
): WorkbenchQuery {
const queryContext: QueryContext = {};
if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
return new WorkbenchQuery({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@ -103,9 +105,7 @@ export class WorkbenchQuery {
arrayMode,
),
).toString(),
queryContext: {
arrayIngestMode: 'array',
},
queryContext,
});
}

19
web-console/src/helpers/spec-conversion.spec.ts
View File

@ -123,10 +123,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 3,
finalizeAggregations: false,
maxNumTasks: 5,
indexSpec: {
dimensionCompression: 'lzf',
@ -232,11 +229,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
finalizeAggregations: false,
});
expect(converted.queryContext).toEqual({});
});
it('converts index_hadoop spec (with rollup)', () => {
@ -357,11 +350,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
finalizeAggregations: false,
});
expect(converted.queryContext).toEqual({});
});
it('converts with issue when there is a __time transform', () => {
@ -663,5 +652,9 @@ describe('spec conversion', () => {
});
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
arrayIngestMode: 'array',
});
});
});

19
web-console/src/helpers/spec-conversion.ts
View File

@ -32,11 +32,18 @@ import type {
DimensionSpec,
IngestionSpec,
MetricSpec,
QueryContext,
QueryWithContext,
TimestampSpec,
Transform,
} from '../druid-models';
import { inflateDimensionSpec, NO_SUCH_COLUMN, TIME_COLUMN, upgradeSpec } from '../druid-models';
import {
getArrayMode,
inflateDimensionSpec,
NO_SUCH_COLUMN,
TIME_COLUMN,
upgradeSpec,
} from '../druid-models';
import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils';
export function getSpecDatasourceName(spec: Partial<IngestionSpec>): string {
@ -73,11 +80,11 @@ export function convertSpecToSql(spec: any): QueryWithContext {
}
spec = upgradeSpec(spec, true);
const context: Record<string, any> = {
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
arrayIngestMode: 'array',
};
const context: QueryContext = {};
if (getArrayMode(spec, 'multi-values') === 'arrays') {
context.arrayIngestMode = 'array';
}
const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
if (indexSpec) {

5
web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
View File

@ -48,7 +48,6 @@ import './sql-data-loader-view.scss';
const INITIAL_QUERY_CONTEXT: QueryContext = {
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
arrayIngestMode: 'array',
};
interface LoaderContent extends QueryWithContext {
@ -190,6 +189,8 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
initInputFormat={inputFormat}
doneButton={false}
onSet={({ inputSource, inputFormat, signature, timeExpression, arrayMode }) => {
const queryContext: QueryContext = { ...INITIAL_QUERY_CONTEXT };
if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
setContent({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@ -199,7 +200,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
arrayMode,
),
).toString(),
queryContext: INITIAL_QUERY_CONTEXT,
queryContext,
});
}}
altText="Skip the wizard and continue with custom SQL"

56
web-console/src/views/workbench-view/run-panel/run-panel.tsx
View File

@ -25,6 +25,7 @@ import {
MenuDivider,
MenuItem,
Position,
Tag,
useHotkeys,
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
@ -35,8 +36,15 @@ import React, { useCallback, useMemo, useState } from 'react';
import { MenuCheckbox, MenuTristate } from '../../../components';
import { EditContextDialog, StringInputDialog } from '../../../dialogs';
import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
import type { DruidEngine, IndexSpec, QueryContext, WorkbenchQuery } from '../../../druid-models';
import type {
ArrayIngestMode,
DruidEngine,
IndexSpec,
QueryContext,
WorkbenchQuery,
} from '../../../druid-models';
import {
changeArrayIngestMode,
changeDurableShuffleStorage,
changeFailOnEmptyInsert,
changeFinalizeAggregations,
@ -47,6 +55,7 @@ import {
changeUseApproximateTopN,
changeUseCache,
changeWaitUntilSegmentsLoad,
getArrayIngestMode,
getDurableShuffleStorage,
getFailOnEmptyInsert,
getFinalizeAggregations,
@ -59,6 +68,7 @@ import {
getWaitUntilSegmentsLoad,
summarizeIndexSpec,
} from '../../../druid-models';
import { getLink } from '../../../links';
import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog';
@ -87,6 +97,20 @@ const NAMED_TIMEZONES: string[] = [
'Australia/Sydney', // +11.0
];
const ARRAY_INGEST_MODE_DESCRIPTION: Record<ArrayIngestMode, JSX.Element> = {
array: (
<>
array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
<Tag minimal>ARRAY&lt;STRING&gt;</Tag>
</>
),
mvd: (
<>
mvd: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
</>
),
};
export interface RunPanelProps {
query: WorkbenchQuery;
onQueryChange(query: WorkbenchQuery): void;
@ -112,6 +136,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const numContextKeys = Object.keys(queryContext).length;
const queryParameters = query.queryParameters;
const arrayIngestMode = getArrayIngestMode(queryContext);
const maxParseExceptions = getMaxParseExceptions(queryContext);
const failOnEmptyInsert = getFailOnEmptyInsert(queryContext);
const finalizeAggregations = getFinalizeAggregations(queryContext);
@ -472,6 +497,35 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
changeQueryContext={changeQueryContext}
/>
)}
{ingestMode && (
<Popover2
position={Position.BOTTOM_LEFT}
content={
<Menu>
{([undefined, 'array', 'mvd'] as (ArrayIngestMode | undefined)[]).map((m, i) => (
<MenuItem
key={i}
icon={tickIcon(m === arrayIngestMode)}
text={m ? ARRAY_INGEST_MODE_DESCRIPTION[m] : '(server default)'}
onClick={() => changeQueryContext(changeArrayIngestMode(queryContext, m))}
/>
))}
<MenuDivider />
<MenuItem
icon={IconNames.HELP}
text="Documentation"
href={`${getLink('DOCS')}/querying/arrays#arrayingestmode`}
target="_blank"
/>
</Menu>
}
>
<Button
text={`Array ingest mode: ${arrayIngestMode ?? '(server default)'}`}
rightIcon={IconNames.CARET_DOWN}
/>
</Popover2>
)}
</ButtonGroup>
)}
{moreMenu && (

9
web-console/src/views/workbench-view/workbench-view.tsx
View File

@ -320,18 +320,13 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
return (
<ConnectExternalDataDialog
onSetExternalConfig={(
externalConfig,
timeExpression,
partitionedByHint,
forceMultiValue,
) => {
onSetExternalConfig={(externalConfig, timeExpression, partitionedByHint, arrayMode) => {
this.handleNewTab(
WorkbenchQuery.fromInitExternalConfig(
externalConfig,
timeExpression,
partitionedByHint,
forceMultiValue,
arrayMode,
),
'Ext ' + guessDataSourceNameFromInputSource(externalConfig.inputSource),
);