diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 5a751af2011..fa3fd905be3 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -297,7 +297,10 @@ export function getSchemaMode(spec: Partial): SchemaMode { return Array.isArray(dimensions) && dimensions.length === 0 ? 'string-only-discovery' : 'fixed'; } -export function getArrayMode(spec: Partial): ArrayMode { +export function getArrayMode( + spec: Partial, + whenUnclear: ArrayMode = 'arrays', +): ArrayMode { const schemaMode = getSchemaMode(spec); switch (schemaMode) { case 'type-aware-discovery': @@ -332,7 +335,7 @@ export function getArrayMode(spec: Partial): ArrayMode { return 'multi-values'; } - return 'arrays'; + return whenUnclear; } } } diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx index 838ab1f3a50..17e8204e949 100644 --- a/web-console/src/druid-models/query-context/query-context.tsx +++ b/web-console/src/druid-models/query-context/query-context.tsx @@ -18,6 +18,8 @@ import { deepDelete, deepSet } from '../../utils'; +export type ArrayIngestMode = 'array' | 'mvd'; + export interface QueryContext { useCache?: boolean; populateCache?: boolean; @@ -32,7 +34,7 @@ export interface QueryContext { durableShuffleStorage?: boolean; maxParseExceptions?: number; groupByEnableMultiValueUnnesting?: boolean; - arrayIngestMode?: 'array' | 'mvd'; + arrayIngestMode?: ArrayIngestMode; [key: string]: any; } @@ -248,3 +250,20 @@ export function changeMaxParseExceptions( return deepDelete(context, 'maxParseExceptions'); } } + +// arrayIngestMode + +export function getArrayIngestMode(context: QueryContext): ArrayIngestMode | undefined { + return context.arrayIngestMode; +} + +export function changeArrayIngestMode( + context: QueryContext, + arrayIngestMode: ArrayIngestMode | undefined, +): QueryContext { + if (arrayIngestMode) { + return deepSet(context, 'arrayIngestMode', arrayIngestMode); + } else { + return deepDelete(context, 'arrayIngestMode'); + } +} diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 27cf6a0109d..d59cdbbe92e 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -94,6 +94,8 @@ export class WorkbenchQuery { partitionedByHint: string | undefined, arrayMode: ArrayMode, ): WorkbenchQuery { + const queryContext: QueryContext = {}; + if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array'; return new WorkbenchQuery({ queryString: ingestQueryPatternToQuery( externalConfigToIngestQueryPattern( @@ -103,9 +105,7 @@ export class WorkbenchQuery { arrayMode, ), ).toString(), - queryContext: { - arrayIngestMode: 'array', - }, + queryContext, }); } diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index bea3f730318..9271196dafa 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -123,10 +123,7 @@ describe('spec conversion', () => { expect(converted.queryString).toMatchSnapshot(); expect(converted.queryContext).toEqual({ - arrayIngestMode: 'array', - groupByEnableMultiValueUnnesting: false, maxParseExceptions: 3, - finalizeAggregations: false, maxNumTasks: 5, indexSpec: { dimensionCompression: 'lzf', @@ -232,11 +229,7 @@ describe('spec conversion', () => { expect(converted.queryString).toMatchSnapshot(); - expect(converted.queryContext).toEqual({ - arrayIngestMode: 'array', - groupByEnableMultiValueUnnesting: false, - finalizeAggregations: false, - }); + expect(converted.queryContext).toEqual({}); }); it('converts index_hadoop spec (with rollup)', () => { @@ -357,11 +350,7 @@ describe('spec conversion', () => { expect(converted.queryString).toMatchSnapshot(); - expect(converted.queryContext).toEqual({ - arrayIngestMode: 'array', - groupByEnableMultiValueUnnesting: false, - finalizeAggregations: false, - }); + expect(converted.queryContext).toEqual({}); }); it('converts with issue when there is a __time transform', () => { @@ -663,5 +652,9 @@ describe('spec conversion', () => { }); expect(converted.queryString).toMatchSnapshot(); + + expect(converted.queryContext).toEqual({ + arrayIngestMode: 'array', + }); }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index f25406e5028..7e7673c7384 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -32,11 +32,18 @@ import type { DimensionSpec, IngestionSpec, MetricSpec, + QueryContext, QueryWithContext, TimestampSpec, Transform, } from '../druid-models'; -import { inflateDimensionSpec, NO_SUCH_COLUMN, TIME_COLUMN, upgradeSpec } from '../druid-models'; +import { + getArrayMode, + inflateDimensionSpec, + NO_SUCH_COLUMN, + TIME_COLUMN, + upgradeSpec, +} from '../druid-models'; import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils'; export function getSpecDatasourceName(spec: Partial): string { @@ -73,11 +80,11 @@ export function convertSpecToSql(spec: any): QueryWithContext { } spec = upgradeSpec(spec, true); - const context: Record = { - finalizeAggregations: false, - groupByEnableMultiValueUnnesting: false, - arrayIngestMode: 'array', - }; + const context: QueryContext = {}; + + if (getArrayMode(spec, 'multi-values') === 'arrays') { + context.arrayIngestMode = 'array'; + } const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec'); if (indexSpec) { diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx index 0fc89573bd2..27017f725da 100644 --- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx +++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx @@ -48,7 +48,6 @@ import './sql-data-loader-view.scss'; const INITIAL_QUERY_CONTEXT: QueryContext = { finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, - arrayIngestMode: 'array', }; interface LoaderContent extends QueryWithContext { @@ -190,6 +189,8 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView( initInputFormat={inputFormat} doneButton={false} onSet={({ inputSource, inputFormat, signature, timeExpression, arrayMode }) => { + const queryContext: QueryContext = { ...INITIAL_QUERY_CONTEXT }; + if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array'; setContent({ queryString: ingestQueryPatternToQuery( externalConfigToIngestQueryPattern( @@ -199,7 +200,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView( arrayMode, ), ).toString(), - queryContext: INITIAL_QUERY_CONTEXT, + queryContext, }); }} altText="Skip the wizard and continue with custom SQL" diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index ac67c40ff27..5d9c9852791 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -25,6 +25,7 @@ import { MenuDivider, MenuItem, Position, + Tag, useHotkeys, } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; @@ -35,8 +36,15 @@ import React, { useCallback, useMemo, useState } from 'react'; import { MenuCheckbox, MenuTristate } from '../../../components'; import { EditContextDialog, StringInputDialog } from '../../../dialogs'; import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog'; -import type { DruidEngine, IndexSpec, QueryContext, WorkbenchQuery } from '../../../druid-models'; +import type { + ArrayIngestMode, + DruidEngine, + IndexSpec, + QueryContext, + WorkbenchQuery, +} from '../../../druid-models'; import { + changeArrayIngestMode, changeDurableShuffleStorage, changeFailOnEmptyInsert, changeFinalizeAggregations, @@ -47,6 +55,7 @@ import { changeUseApproximateTopN, changeUseCache, changeWaitUntilSegmentsLoad, + getArrayIngestMode, getDurableShuffleStorage, getFailOnEmptyInsert, getFinalizeAggregations, @@ -59,6 +68,7 @@ import { getWaitUntilSegmentsLoad, summarizeIndexSpec, } from '../../../druid-models'; +import { getLink } from '../../../links'; import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils'; import { MaxTasksButton } from '../max-tasks-button/max-tasks-button'; import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog'; @@ -87,6 +97,20 @@ const NAMED_TIMEZONES: string[] = [ 'Australia/Sydney', // +11.0 ]; +const ARRAY_INGEST_MODE_DESCRIPTION: Record = { + array: ( + <> + array: Load SQL VARCHAR ARRAY as Druid{' '} + ARRAY<STRING> + + ), + mvd: ( + <> + mvd: Load SQL VARCHAR ARRAY as Druid multi-value STRING + + ), +}; + export interface RunPanelProps { query: WorkbenchQuery; onQueryChange(query: WorkbenchQuery): void; @@ -112,6 +136,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const numContextKeys = Object.keys(queryContext).length; const queryParameters = query.queryParameters; + const arrayIngestMode = getArrayIngestMode(queryContext); const maxParseExceptions = getMaxParseExceptions(queryContext); const failOnEmptyInsert = getFailOnEmptyInsert(queryContext); const finalizeAggregations = getFinalizeAggregations(queryContext); @@ -472,6 +497,35 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { changeQueryContext={changeQueryContext} /> )} + {ingestMode && ( + + {([undefined, 'array', 'mvd'] as (ArrayIngestMode | undefined)[]).map((m, i) => ( + changeQueryContext(changeArrayIngestMode(queryContext, m))} + /> + ))} + + + + } + > +