Web console: Make array ingest mode ux better (#15927)

* only set arrayIngestMode: array when needed (still do the queries the correct way)

* arrayIngestMode control

* update wording

* feedback fixes

web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx

@@ -297,7 +297,10 @@ export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
   return Array.isArray(dimensions) && dimensions.length === 0 ? 'string-only-discovery' : 'fixed';
 }
 
-export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
+export function getArrayMode(
+  spec: Partial<IngestionSpec>,
+  whenUnclear: ArrayMode = 'arrays',
+): ArrayMode {
   const schemaMode = getSchemaMode(spec);
   switch (schemaMode) {
     case 'type-aware-discovery':
@@ -332,7 +335,7 @@ export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
         return 'multi-values';
       }
 
-      return 'arrays';
+      return whenUnclear;
     }
   }
 }

web-console/src/druid-models/query-context/query-context.tsx

@@ -18,6 +18,8 @@
 
 import { deepDelete, deepSet } from '../../utils';
 
+export type ArrayIngestMode = 'array' | 'mvd';
+
 export interface QueryContext {
   useCache?: boolean;
   populateCache?: boolean;
@@ -32,7 +34,7 @@ export interface QueryContext {
   durableShuffleStorage?: boolean;
   maxParseExceptions?: number;
   groupByEnableMultiValueUnnesting?: boolean;
-  arrayIngestMode?: 'array' | 'mvd';
+  arrayIngestMode?: ArrayIngestMode;
 
   [key: string]: any;
 }
@@ -248,3 +250,20 @@ export function changeMaxParseExceptions(
     return deepDelete(context, 'maxParseExceptions');
   }
 }
+
+// arrayIngestMode
+
+export function getArrayIngestMode(context: QueryContext): ArrayIngestMode | undefined {
+  return context.arrayIngestMode;
+}
+
+export function changeArrayIngestMode(
+  context: QueryContext,
+  arrayIngestMode: ArrayIngestMode | undefined,
+): QueryContext {
+  if (arrayIngestMode) {
+    return deepSet(context, 'arrayIngestMode', arrayIngestMode);
+  } else {
+    return deepDelete(context, 'arrayIngestMode');
+  }
+}

web-console/src/druid-models/workbench-query/workbench-query.ts

@@ -94,6 +94,8 @@ export class WorkbenchQuery {
     partitionedByHint: string | undefined,
     arrayMode: ArrayMode,
   ): WorkbenchQuery {
+    const queryContext: QueryContext = {};
+    if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
     return new WorkbenchQuery({
       queryString: ingestQueryPatternToQuery(
         externalConfigToIngestQueryPattern(
@@ -103,9 +105,7 @@ export class WorkbenchQuery {
           arrayMode,
         ),
       ).toString(),
-      queryContext: {
+      queryContext,
-        arrayIngestMode: 'array',
-      },
     });
   }
 

web-console/src/helpers/spec-conversion.spec.ts

@@ -123,10 +123,7 @@ describe('spec conversion', () => {
     expect(converted.queryString).toMatchSnapshot();
 
     expect(converted.queryContext).toEqual({
-      arrayIngestMode: 'array',
-      groupByEnableMultiValueUnnesting: false,
       maxParseExceptions: 3,
-      finalizeAggregations: false,
       maxNumTasks: 5,
       indexSpec: {
         dimensionCompression: 'lzf',
@@ -232,11 +229,7 @@ describe('spec conversion', () => {
 
     expect(converted.queryString).toMatchSnapshot();
 
-    expect(converted.queryContext).toEqual({
+    expect(converted.queryContext).toEqual({});
-      arrayIngestMode: 'array',
-      groupByEnableMultiValueUnnesting: false,
-      finalizeAggregations: false,
-    });
   });
 
   it('converts index_hadoop spec (with rollup)', () => {
@@ -357,11 +350,7 @@ describe('spec conversion', () => {
 
     expect(converted.queryString).toMatchSnapshot();
 
-    expect(converted.queryContext).toEqual({
+    expect(converted.queryContext).toEqual({});
-      arrayIngestMode: 'array',
-      groupByEnableMultiValueUnnesting: false,
-      finalizeAggregations: false,
-    });
   });
 
   it('converts with issue when there is a __time transform', () => {
@@ -663,5 +652,9 @@ describe('spec conversion', () => {
     });
 
     expect(converted.queryString).toMatchSnapshot();
+
+    expect(converted.queryContext).toEqual({
+      arrayIngestMode: 'array',
+    });
   });
 });

web-console/src/helpers/spec-conversion.ts

@@ -32,11 +32,18 @@ import type {
   DimensionSpec,
   IngestionSpec,
   MetricSpec,
+  QueryContext,
   QueryWithContext,
   TimestampSpec,
   Transform,
 } from '../druid-models';
-import { inflateDimensionSpec, NO_SUCH_COLUMN, TIME_COLUMN, upgradeSpec } from '../druid-models';
+import {
+  getArrayMode,
+  inflateDimensionSpec,
+  NO_SUCH_COLUMN,
+  TIME_COLUMN,
+  upgradeSpec,
+} from '../druid-models';
 import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils';
 
 export function getSpecDatasourceName(spec: Partial<IngestionSpec>): string {
@@ -73,11 +80,11 @@ export function convertSpecToSql(spec: any): QueryWithContext {
   }
   spec = upgradeSpec(spec, true);
 
-  const context: Record<string, any> = {
+  const context: QueryContext = {};
-    finalizeAggregations: false,
+
-    groupByEnableMultiValueUnnesting: false,
+  if (getArrayMode(spec, 'multi-values') === 'arrays') {
-    arrayIngestMode: 'array',
+    context.arrayIngestMode = 'array';
-  };
+  }
 
   const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
   if (indexSpec) {

web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx

@@ -48,7 +48,6 @@ import './sql-data-loader-view.scss';
 const INITIAL_QUERY_CONTEXT: QueryContext = {
   finalizeAggregations: false,
   groupByEnableMultiValueUnnesting: false,
-  arrayIngestMode: 'array',
 };
 
 interface LoaderContent extends QueryWithContext {
@@ -190,6 +189,8 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
             initInputFormat={inputFormat}
             doneButton={false}
             onSet={({ inputSource, inputFormat, signature, timeExpression, arrayMode }) => {
+              const queryContext: QueryContext = { ...INITIAL_QUERY_CONTEXT };
+              if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
               setContent({
                 queryString: ingestQueryPatternToQuery(
                   externalConfigToIngestQueryPattern(
@@ -199,7 +200,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
                     arrayMode,
                   ),
                 ).toString(),
-                queryContext: INITIAL_QUERY_CONTEXT,
+                queryContext,
               });
             }}
             altText="Skip the wizard and continue with custom SQL"

web-console/src/views/workbench-view/run-panel/run-panel.tsx

@@ -25,6 +25,7 @@ import {
   MenuDivider,
   MenuItem,
   Position,
+  Tag,
   useHotkeys,
 } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
@@ -35,8 +36,15 @@ import React, { useCallback, useMemo, useState } from 'react';
 import { MenuCheckbox, MenuTristate } from '../../../components';
 import { EditContextDialog, StringInputDialog } from '../../../dialogs';
 import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
-import type { DruidEngine, IndexSpec, QueryContext, WorkbenchQuery } from '../../../druid-models';
+import type {
+  ArrayIngestMode,
+  DruidEngine,
+  IndexSpec,
+  QueryContext,
+  WorkbenchQuery,
+} from '../../../druid-models';
 import {
+  changeArrayIngestMode,
   changeDurableShuffleStorage,
   changeFailOnEmptyInsert,
   changeFinalizeAggregations,
@@ -47,6 +55,7 @@ import {
   changeUseApproximateTopN,
   changeUseCache,
   changeWaitUntilSegmentsLoad,
+  getArrayIngestMode,
   getDurableShuffleStorage,
   getFailOnEmptyInsert,
   getFinalizeAggregations,
@@ -59,6 +68,7 @@ import {
   getWaitUntilSegmentsLoad,
   summarizeIndexSpec,
 } from '../../../druid-models';
+import { getLink } from '../../../links';
 import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
 import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
 import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog';
@@ -87,6 +97,20 @@ const NAMED_TIMEZONES: string[] = [
   'Australia/Sydney', // +11.0
 ];
 
+const ARRAY_INGEST_MODE_DESCRIPTION: Record<ArrayIngestMode, JSX.Element> = {
+  array: (
+    <>
+      array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
+      <Tag minimal>ARRAY&lt;STRING&gt;</Tag>
+    </>
+  ),
+  mvd: (
+    <>
+      mvd: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
+    </>
+  ),
+};
+
 export interface RunPanelProps {
   query: WorkbenchQuery;
   onQueryChange(query: WorkbenchQuery): void;
@@ -112,6 +136,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
   const numContextKeys = Object.keys(queryContext).length;
   const queryParameters = query.queryParameters;
 
+  const arrayIngestMode = getArrayIngestMode(queryContext);
   const maxParseExceptions = getMaxParseExceptions(queryContext);
   const failOnEmptyInsert = getFailOnEmptyInsert(queryContext);
   const finalizeAggregations = getFinalizeAggregations(queryContext);
@@ -472,6 +497,35 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
               changeQueryContext={changeQueryContext}
             />
           )}
+          {ingestMode && (
+            <Popover2
+              position={Position.BOTTOM_LEFT}
+              content={
+                <Menu>
+                  {([undefined, 'array', 'mvd'] as (ArrayIngestMode | undefined)[]).map((m, i) => (
+                    <MenuItem
+                      key={i}
+                      icon={tickIcon(m === arrayIngestMode)}
+                      text={m ? ARRAY_INGEST_MODE_DESCRIPTION[m] : '(server default)'}
+                      onClick={() => changeQueryContext(changeArrayIngestMode(queryContext, m))}
+                    />
+                  ))}
+                  <MenuDivider />
+                  <MenuItem
+                    icon={IconNames.HELP}
+                    text="Documentation"
+                    href={`${getLink('DOCS')}/querying/arrays#arrayingestmode`}
+                    target="_blank"
+                  />
+                </Menu>
+              }
+            >
+              <Button
+                text={`Array ingest mode: ${arrayIngestMode ?? '(server default)'}`}
+                rightIcon={IconNames.CARET_DOWN}
+              />
+            </Popover2>
+          )}
         </ButtonGroup>
       )}
       {moreMenu && (

web-console/src/views/workbench-view/workbench-view.tsx

@@ -320,18 +320,13 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
 
     return (
       <ConnectExternalDataDialog
-        onSetExternalConfig={(
+        onSetExternalConfig={(externalConfig, timeExpression, partitionedByHint, arrayMode) => {
-          externalConfig,
-          timeExpression,
-          partitionedByHint,
-          forceMultiValue,
-        ) => {
           this.handleNewTab(
             WorkbenchQuery.fromInitExternalConfig(
               externalConfig,
               timeExpression,
               partitionedByHint,
-              forceMultiValue,
+              arrayMode,
             ),
             'Ext ' + guessDataSourceNameFromInputSource(externalConfig.inputSource),
           );