[backport] Backport 24.0.1 web console issue fixes (#13146)

* fix number of expected functions (#13050)

* default to no compare (#13041)

* quote columns, datasources in auto complete if needed (#13060)

* Web console: better detection for arrays containing objects (#13077)

* better detection for arrays containing objects

* include boolean also

* link to error docs (#13094)

* Web console: correctly escape path based flatten specs (#13105)

* fix path generation

* do escape

* fix replace

* fix replace for good

* append to exisitng callout (#13130)

* better spec conversion with issues (#13136)

* bump version to 24.0.1

web-console/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "web-console",
-  "version": "24.0.0",
+  "version": "24.0.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

web-console/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-console",
-  "version": "24.0.0",
+  "version": "24.0.1",
   "description": "A web console for Apache Druid",
   "author": "Apache Druid Developers <dev@druid.apache.org>",
   "license": "Apache-2.0",

web-console/script/create-sql-docs.js

@@ -23,7 +23,7 @@ const snarkdown = require('snarkdown');
 
 const writefile = 'lib/sql-docs.js';
 
-const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 158;
+const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162;
 const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
 
 function hasHtmlTags(str) {
@@ -90,15 +90,15 @@ const readDoc = async () => {
 
   // Make sure there are enough functions found
   const numFunction = Object.keys(functionDocs).length;
-  if (numFunction < MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS) {
+  if (!(MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS <= numFunction)) {
     throw new Error(
       `Did not find enough function entries did the structure of '${readfile}' change? (found ${numFunction} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS})`,
     );
   }
 
   // Make sure there are at least 10 data types for sanity
-  const numDataTypes = dataTypeDocs.length;
+  const numDataTypes = Object.keys(dataTypeDocs).length;
-  if (numDataTypes < MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES) {
+  if (!(MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES <= numDataTypes)) {
     throw new Error(
       `Did not find enough data type entries did the structure of '${readfile}' change? (found ${numDataTypes} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES})`,
     );

web-console/src/ace-modes/dsql.js

@@ -63,6 +63,10 @@ ace.define(
 
       this.$rules = {
         start: [
+          {
+            token: 'comment.issue',
+            regex: '--:ISSUE:.*$',
+          },
           {
             token: 'comment',
             regex: '--.*$',
@@ -73,17 +77,13 @@ ace.define(
             end: '\\*/',
           },
           {
-            token: 'string', // " string
+            token: 'variable.column', // " quoted reference
             regex: '".*?"',
           },
           {
-            token: 'string', // ' string
+            token: 'string', // ' string literal
             regex: "'.*?'",
           },
-          {
-            token: 'string', // ` string (apache drill)
-            regex: '`.*?`',
-          },
           {
             token: 'constant.numeric', // float
             regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',

web-console/src/bootstrap/ace.scss

@@ -25,6 +25,18 @@
 .ace-solarized-dark {
   background-color: rgba($dark-gray1, 0.5);
 
+  // START: Custom code styles
+  .ace_variable.ace_column {
+    color: #2ceefb;
+  }
+
+  .ace_comment.ace_issue {
+    color: #cb3116;
+    text-decoration: underline;
+    text-decoration-style: wavy;
+  }
+  // END: Custom code styles
+
   &.no-background {
     background-color: transparent;
   }

web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap

@@ -326,7 +326,7 @@ exports[`HeaderBar matches snapshot 1`] = `
           <Blueprint4.MenuItem
             active={false}
             disabled={false}
-            href="https://druid.apache.org/docs/24.0.0"
+            href="https://druid.apache.org/docs/24.0.1"
             icon="th"
             multiline={false}
             popoverProps={Object {}}

web-console/src/components/table-cell/__snapshots__/table-cell.spec.tsx.snap

@@ -49,6 +49,14 @@ exports[`TableCell matches snapshot array long 1`] = `
 </div>
 `;
 
+exports[`TableCell matches snapshot array mixed 1`] = `
+<div
+  class="table-cell plain"
+>
+  ["a",{"v":"b"},"c"]
+</div>
+`;
+
 exports[`TableCell matches snapshot array short 1`] = `
 <div
   class="table-cell plain"

web-console/src/components/table-cell/table-cell.spec.tsx

@@ -64,6 +64,13 @@ describe('TableCell', () => {
     expect(container.firstChild).toMatchSnapshot();
   });
 
+  it('matches snapshot array mixed', () => {
+    const tableCell = <TableCell value={['a', { v: 'b' }, 'c']} />;
+
+    const { container } = render(tableCell);
+    expect(container.firstChild).toMatchSnapshot();
+  });
+
   it('matches snapshot object', () => {
     const tableCell = <TableCell value={{ hello: 'world' }} />;
 

web-console/src/components/table-cell/table-cell.tsx

@@ -21,6 +21,7 @@ import * as JSONBig from 'json-bigint-native';
 import React, { useState } from 'react';
 
 import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
+import { isSimpleArray } from '../../utils';
 import { ActionIcon } from '../action-icon/action-icon';
 
 import './table-cell.scss';
@@ -97,7 +98,7 @@ export const TableCell = React.memo(function TableCell(props: TableCellProps) {
           {isNaN(dateValue) ? 'Unusable date' : value.toISOString()}
         </div>
       );
-    } else if (Array.isArray(value)) {
+    } else if (isSimpleArray(value)) {
       return renderTruncated(`[${value.join(', ')}]`);
     } else if (typeof value === 'object') {
       return renderTruncated(JSONBig.stringify(value));

web-console/src/components/table-filterable-cell/table-filterable-cell.tsx

@@ -34,14 +34,14 @@ export interface TableFilterableCellProps {
   value: string;
   filters: Filter[];
   onFiltersChange(filters: Filter[]): void;
-  disableComparisons?: boolean;
+  enableComparisons?: boolean;
   children?: ReactNode;
 }
 
 export const TableFilterableCell = React.memo(function TableFilterableCell(
   props: TableFilterableCellProps,
 ) {
-  const { field, value, children, filters, disableComparisons, onFiltersChange } = props;
+  const { field, value, children, filters, enableComparisons, onFiltersChange } = props;
 
   return (
     <Popover2
@@ -51,7 +51,7 @@ export const TableFilterableCell = React.memo(function TableFilterableCell(
           content={() => (
             <Menu>
               <MenuDivider title="Filter" />
-              {(disableComparisons ? FILTER_MODES_NO_COMPARISONS : FILTER_MODES).map((mode, i) => (
+              {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISONS).map((mode, i) => (
                 <MenuItem
                   key={i}
                   icon={filterModeToIcon(mode)}

web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap

@@ -12,7 +12,7 @@ exports[`CoordinatorDynamicConfigDialog matches snapshot 1`] = `
     Edit the coordinator dynamic configuration on the fly. For more information please refer to the
      
     <Memo(ExternalLink)
-      href="https://druid.apache.org/docs/24.0.0/configuration/index.html#dynamic-configuration"
+      href="https://druid.apache.org/docs/24.0.1/configuration/index.html#dynamic-configuration"
     >
       documentation
     </Memo(ExternalLink)>

web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap

@@ -11,7 +11,7 @@ exports[`OverlordDynamicConfigDialog matches snapshot 1`] = `
     Edit the overlord dynamic configuration on the fly. For more information please refer to the
      
     <Memo(ExternalLink)
-      href="https://druid.apache.org/docs/24.0.0/configuration/index.html#overlord-dynamic-configuration"
+      href="https://druid.apache.org/docs/24.0.1/configuration/index.html#overlord-dynamic-configuration"
     >
       documentation
     </Memo(ExternalLink)>

web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap

@@ -63,7 +63,7 @@ exports[`RetentionDialog matches snapshot 1`] = `
             Druid uses rules to determine what data should be retained in the cluster. The rules are evaluated in order from top to bottom. For more information please refer to the
              
             <a
-              href="https://druid.apache.org/docs/24.0.0/operations/rule-configuration.html"
+              href="https://druid.apache.org/docs/24.0.1/operations/rule-configuration.html"
               rel="noopener noreferrer"
               target="_blank"
             >

web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts

@@ -22,7 +22,7 @@ describe('flatten-spec', () => {
   describe('computeFlattenExprsForData', () => {
     const data = [
       {
-        context: { host: 'cla', topic: 'moon', bonus: { foo: 'bar' } },
+        context: { host: 'cla', topic: 'moon', bonus: { 'fo.o': 'bar' } },
         tags: ['a', 'b', 'c'],
         messages: [
           { metric: 'request/time', value: 122 },
@@ -32,7 +32,7 @@ describe('flatten-spec', () => {
         value: 5,
       },
       {
-        context: { host: 'piv', popic: 'sun' },
+        context: { 'host': 'piv', '1pic': 'sun' },
         tags: ['a', 'd'],
         messages: [
           { metric: 'request/time', value: 44 },
@@ -41,7 +41,7 @@ describe('flatten-spec', () => {
         value: 4,
       },
       {
-        context: { host: 'imp', dopik: 'fun' },
+        context: { 'host': 'imp', "d\\o\npi'c'": 'fun' },
         tags: ['x', 'y'],
         messages: [
           { metric: 'request/time', value: 4 },
@@ -53,22 +53,12 @@ describe('flatten-spec', () => {
     ];
 
     it('works for path, ignore-arrays', () => {
-      expect(computeFlattenExprsForData(data, 'path', 'ignore-arrays')).toEqual([
+      expect(computeFlattenExprsForData(data, 'ignore-arrays')).toEqual([
-        '$.context.bonus.foo',
+        "$.context.bonus['fo.o']",
-        '$.context.dopik',
         '$.context.host',
-        '$.context.popic',
         '$.context.topic',
-      ]);
+        "$.context['1pic']",
-    });
+        "$.context['d\\\\o\npi\\'c\\'']",
-
-    it('works for jq, ignore-arrays', () => {
-      expect(computeFlattenExprsForData(data, 'jq', 'ignore-arrays')).toEqual([
-        '.context.bonus.foo',
-        '.context.dopik',
-        '.context.host',
-        '.context.popic',
-        '.context.topic',
       ]);
     });
   });

web-console/src/druid-models/flatten-spec/flatten-spec.tsx

@@ -61,18 +61,22 @@ export const FLATTEN_FIELD_FIELDS: Field<FlattenField>[] = [
   },
 ];
 
-export type ExprType = 'path' | 'jq';
 export type ArrayHandling = 'ignore-arrays' | 'include-arrays';
 
+function escapePathKey(pathKey: string): string {
+  return /^[a-z]\w*$/i.test(pathKey)
+    ? `.${pathKey}`
+    : `['${pathKey.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}']`;
+}
+
 export function computeFlattenPathsForData(
   data: Record<string, any>[],
-  exprType: ExprType,
   arrayHandling: ArrayHandling,
 ): FlattenField[] {
-  return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => {
+  return computeFlattenExprsForData(data, arrayHandling).map(expr => {
     return {
-      name: expr.replace(/^\$?\./, ''),
+      name: expr.replace(/^\$\./, '').replace(/['\]]/g, '').replace(/\[/g, '.'),
-      type: exprType,
+      type: 'path',
       expr,
     };
   });
@@ -80,7 +84,6 @@ export function computeFlattenPathsForData(
 
 export function computeFlattenExprsForData(
   data: Record<string, any>[],
-  exprType: ExprType,
   arrayHandling: ArrayHandling,
   includeTopLevel = false,
 ): string[] {
@@ -91,12 +94,7 @@ export function computeFlattenExprsForData(
     for (const datumKey of datumKeys) {
       const datumValue = datum[datumKey];
       if (includeTopLevel || isNested(datumValue)) {
-        addPath(
+        addPath(seenPaths, `$${escapePathKey(datumKey)}`, datumValue, arrayHandling);
-          seenPaths,
-          exprType === 'path' ? `$.${datumKey}` : `.${datumKey}`,
-          datumValue,
-          arrayHandling,
-        );
       }
     }
   }
@@ -114,7 +112,7 @@ function addPath(
     if (!Array.isArray(value)) {
       const valueKeys = Object.keys(value);
       for (const valueKey of valueKeys) {
-        addPath(paths, `${path}.${valueKey}`, value[valueKey], arrayHandling);
+        addPath(paths, `${path}${escapePathKey(valueKey)}`, value[valueKey], arrayHandling);
       }
     } else if (arrayHandling === 'include-arrays') {
       for (let i = 0; i < value.length; i++) {

web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts

@@ -725,6 +725,15 @@ describe('spec utils', () => {
     it('works for multi-value', () => {
       expect(guessColumnTypeFromInput(['a', ['b'], 'c'], false)).toEqual('string');
       expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string');
+      expect(guessColumnTypeFromInput([true, [true, 7, false], false, 'x'], false)).toEqual(
+        'string',
+      );
+    });
+
+    it('works for complex arrays', () => {
+      expect(guessColumnTypeFromInput([{ type: 'Dogs' }, { type: 'JavaScript' }], false)).toEqual(
+        'COMPLEX<json>',
+      );
     });
 
     it('works for strange json', () => {

web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx

@@ -32,6 +32,7 @@ import {
   EMPTY_ARRAY,
   EMPTY_OBJECT,
   filterMap,
+  isSimpleArray,
   oneOf,
   parseCsvLine,
   typeIs,
@@ -2309,7 +2310,7 @@ export function guessIsArrayFromHeaderAndRows(
   headerAndRows: SampleHeaderAndRows,
   column: string,
 ): boolean {
-  return headerAndRows.rows.some(r => Array.isArray(r.input?.[column]));
+  return headerAndRows.rows.some(r => isSimpleArray(r.input?.[column]));
 }
 
 export function guessColumnTypeFromInput(
@@ -2322,7 +2323,7 @@ export function guessColumnTypeFromInput(
   if (!definedValues.length) return 'string';
 
   // If we see any arrays in the input this is a multi-value dimension that must be a string
-  if (definedValues.some(v => Array.isArray(v))) return 'string';
+  if (definedValues.some(v => isSimpleArray(v))) return 'string';
 
   // If we see any JSON objects in the input assume COMPLEX<json>
   if (definedValues.some(v => v && typeof v === 'object')) return 'COMPLEX<json>';

web-console/src/druid-models/workbench-query/workbench-query.spec.ts

@@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => {
         sqlPrefixLines: 0,
       });
     });
+
+    it('works with sql with ISSUE comment', () => {
+      const sql = sane`
+        SELECT *
+        --:ISSUE: There is something wrong with this query.
+        FROM wikipedia
+      `;
+
+      const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql);
+
+      expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow(
+        `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
+      );
+    });
   });
 
   describe('#getIngestDatasource', () => {

web-console/src/druid-models/workbench-query/workbench-query.ts

@@ -576,10 +576,15 @@ export class WorkbenchQuery {
       apiQuery.query = queryPrepend + apiQuery.query + queryAppend;
     }
 
-    const m = /(--:context\s.+)(?:\n|$)/.exec(apiQuery.query);
+    const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query);
     if (m) {
       throw new Error(
-        `This query contains a context comment '${m[1]}'. Context comments have been deprecated. Please rewrite the context comment as a context parameter. The context parameter editor is located in the "Engine" dropdown.`,
+        `This query contains an ISSUE comment: ${m[1]
+          .trim()
+          .replace(
+            /\.$/,
+            '',
+          )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
       );
     }
 

web-console/src/helpers/spec-conversion.spec.ts

@@ -449,4 +449,262 @@ describe('spec conversion', () => {
       finalizeAggregations: false,
     });
   });
+
+  it('converts with issue when there is a __time transform', () => {
+    const converted = convertSpecToSql({
+      type: 'index_parallel',
+      spec: {
+        ioConfig: {
+          type: 'index_parallel',
+          inputSource: {
+            type: 'http',
+            uris: ['https://druid.apache.org/data/wikipedia.json.gz'],
+          },
+          inputFormat: {
+            type: 'json',
+          },
+        },
+        dataSchema: {
+          granularitySpec: {
+            segmentGranularity: 'hour',
+            queryGranularity: 'none',
+            rollup: false,
+          },
+          dataSource: 'wikipedia',
+          transformSpec: {
+            transforms: [{ name: '__time', expression: '_some_time_parse_expression_' }],
+          },
+          timestampSpec: {
+            column: 'timestamp',
+            format: 'auto',
+          },
+          dimensionsSpec: {
+            dimensions: [
+              'isRobot',
+              'channel',
+              'flags',
+              'isUnpatrolled',
+              'page',
+              'diffUrl',
+              {
+                type: 'long',
+                name: 'added',
+              },
+              'comment',
+              {
+                type: 'long',
+                name: 'commentLength',
+              },
+              'isNew',
+              'isMinor',
+              {
+                type: 'long',
+                name: 'delta',
+              },
+              'isAnonymous',
+              'user',
+              {
+                type: 'long',
+                name: 'deltaBucket',
+              },
+              {
+                type: 'long',
+                name: 'deleted',
+              },
+              'namespace',
+              'cityName',
+              'countryName',
+              'regionIsoCode',
+              'metroCode',
+              'countryIsoCode',
+              'regionName',
+            ],
+          },
+        },
+        tuningConfig: {
+          type: 'index_parallel',
+          partitionsSpec: {
+            type: 'single_dim',
+            partitionDimension: 'isRobot',
+            targetRowsPerSegment: 150000,
+          },
+          forceGuaranteedRollup: true,
+          maxNumConcurrentSubTasks: 4,
+          maxParseExceptions: 3,
+        },
+      },
+    });
+
+    expect(converted.queryString).toEqual(sane`
+      -- This SQL query was auto generated from an ingestion spec
+      REPLACE INTO wikipedia OVERWRITE ALL
+      WITH source AS (SELECT * FROM TABLE(
+        EXTERN(
+          '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
+          '{"type":"json"}',
+          '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
+        )
+      ))
+      SELECT
+        --:ISSUE: The spec contained transforms that could not be automatically converted.
+        REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
+        "isRobot",
+        "channel",
+        "flags",
+        "isUnpatrolled",
+        "page",
+        "diffUrl",
+        "added",
+        "comment",
+        "commentLength",
+        "isNew",
+        "isMinor",
+        "delta",
+        "isAnonymous",
+        "user",
+        "deltaBucket",
+        "deleted",
+        "namespace",
+        "cityName",
+        "countryName",
+        "regionIsoCode",
+        "metroCode",
+        "countryIsoCode",
+        "regionName"
+      FROM source
+      PARTITIONED BY HOUR
+      CLUSTERED BY "isRobot"
+    `);
+  });
+
+  it('converts with issue when there is a dimension transform and strange filter', () => {
+    const converted = convertSpecToSql({
+      type: 'index_parallel',
+      spec: {
+        ioConfig: {
+          type: 'index_parallel',
+          inputSource: {
+            type: 'http',
+            uris: ['https://druid.apache.org/data/wikipedia.json.gz'],
+          },
+          inputFormat: {
+            type: 'json',
+          },
+        },
+        dataSchema: {
+          granularitySpec: {
+            segmentGranularity: 'hour',
+            queryGranularity: 'none',
+            rollup: false,
+          },
+          dataSource: 'wikipedia',
+          transformSpec: {
+            transforms: [{ name: 'comment', expression: '_some_expression_' }],
+            filter: {
+              type: 'strange',
+            },
+          },
+          timestampSpec: {
+            column: 'timestamp',
+            format: 'auto',
+          },
+          dimensionsSpec: {
+            dimensions: [
+              'isRobot',
+              'channel',
+              'flags',
+              'isUnpatrolled',
+              'page',
+              'diffUrl',
+              {
+                type: 'long',
+                name: 'added',
+              },
+              'comment',
+              {
+                type: 'long',
+                name: 'commentLength',
+              },
+              'isNew',
+              'isMinor',
+              {
+                type: 'long',
+                name: 'delta',
+              },
+              'isAnonymous',
+              'user',
+              {
+                type: 'long',
+                name: 'deltaBucket',
+              },
+              {
+                type: 'long',
+                name: 'deleted',
+              },
+              'namespace',
+              'cityName',
+              'countryName',
+              'regionIsoCode',
+              'metroCode',
+              'countryIsoCode',
+              'regionName',
+            ],
+          },
+        },
+        tuningConfig: {
+          type: 'index_parallel',
+          partitionsSpec: {
+            type: 'single_dim',
+            partitionDimension: 'isRobot',
+            targetRowsPerSegment: 150000,
+          },
+          forceGuaranteedRollup: true,
+          maxNumConcurrentSubTasks: 4,
+          maxParseExceptions: 3,
+        },
+      },
+    });
+
+    expect(converted.queryString).toEqual(sane`
+      -- This SQL query was auto generated from an ingestion spec
+      REPLACE INTO wikipedia OVERWRITE ALL
+      WITH source AS (SELECT * FROM TABLE(
+        EXTERN(
+          '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
+          '{"type":"json"}',
+          '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
+        )
+      ))
+      SELECT
+        --:ISSUE: The spec contained transforms that could not be automatically converted.
+        CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
+        "isRobot",
+        "channel",
+        "flags",
+        "isUnpatrolled",
+        "page",
+        "diffUrl",
+        "added",
+        REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
+        "commentLength",
+        "isNew",
+        "isMinor",
+        "delta",
+        "isAnonymous",
+        "user",
+        "deltaBucket",
+        "deleted",
+        "namespace",
+        "cityName",
+        "countryName",
+        "regionIsoCode",
+        "metroCode",
+        "countryIsoCode",
+        "regionName"
+      FROM source
+      WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
+      PARTITIONED BY HOUR
+      CLUSTERED BY "isRobot"
+    `);
+  });
 });

web-console/src/helpers/spec-conversion.ts

@@ -102,10 +102,19 @@ export function convertSpecToSql(spec: any): QueryWithContext {
     );
   }
 
+  const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || [];
+  if (!Array.isArray(transforms)) {
+    throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`);
+  }
+
   let timeExpression: string;
   const column = timestampSpec.column || 'timestamp';
   const columnRef = SqlRef.column(column);
   const format = timestampSpec.format || 'auto';
+  const timeTransform = transforms.find(t => t.name === '__time');
+  if (timeTransform) {
+    timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
+  } else {
     switch (format) {
       case 'auto':
         columns.unshift({ name: column, type: 'string' });
@@ -142,6 +151,7 @@ export function convertSpecToSql(spec: any): QueryWithContext {
         timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`;
         break;
     }
+  }
 
   if (timestampSpec.missingValue) {
     timeExpression = `COALESCE(${timeExpression}, TIME_PARSE(${SqlLiteral.create(
@@ -238,19 +248,24 @@ export function convertSpecToSql(spec: any): QueryWithContext {
 
   lines.push(`SELECT`);
 
-  const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || [];
-  if (!Array.isArray(transforms))
-    throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`);
   if (transforms.length) {
-    lines.push(`  -- The spec contained transforms that could not be automatically converted.`);
+    lines.push(
+      `  --:ISSUE: The spec contained transforms that could not be automatically converted.`,
+    );
   }
 
-  const dimensionExpressions = [`  ${timeExpression} AS __time,`].concat(
+  const dimensionExpressions = [
+    `  ${timeExpression} AS __time,${
+      timeTransform ? ` --:ISSUE: Transform for __time could not be converted` : ''
+    }`,
+  ].concat(
     dimensions.flatMap((dimension: DimensionSpec) => {
       const dimensionName = dimension.name;
       const relevantTransform = transforms.find(t => t.name === dimensionName);
-      return `  ${SqlRef.columnWithQuotes(dimensionName)},${
+      return `  ${
-        relevantTransform ? ` -- Relevant transform: ${JSONBig.stringify(relevantTransform)}` : ''
+        relevantTransform ? `REWRITE_[${relevantTransform.expression}]_TO_SQL AS ` : ''
+      }${SqlRef.columnWithQuotes(dimensionName)},${
+        relevantTransform ? ` --:ISSUE: Transform for dimension could not be converted` : ''
       }`;
     }),
   );
@@ -275,9 +290,9 @@ export function convertSpecToSql(spec: any): QueryWithContext {
       lines.push(`WHERE ${convertFilter(filter)}`);
     } catch {
       lines.push(
-        `-- The spec contained a filter that could not be automatically converted: ${JSONBig.stringify(
+        `WHERE REWRITE_[${JSONBig.stringify(
           filter,
-        )}`,
+        )}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually`,
       );
     }
   }

web-console/src/links.ts

@@ -19,7 +19,7 @@
 import hasOwnProp from 'has-own-prop';
 
 // This is set to the latest available version and should be updated to the next version before release
-const DRUID_DOCS_VERSION = '24.0.0';
+const DRUID_DOCS_VERSION = '24.0.1';
 
 function fillVersion(str: string): string {
   return str.replace(/\{\{VERSION}}/g, DRUID_DOCS_VERSION);
@@ -63,6 +63,7 @@ export type LinkNames =
   | 'DOCS_SQL'
   | 'DOCS_RUNE'
   | 'DOCS_API'
+  | 'DOCS_MSQ_ERROR'
   | 'COMMUNITY'
   | 'SLACK'
   | 'USER_GROUP'
@@ -82,6 +83,8 @@ export function getLink(linkName: LinkNames): string {
       return `${links.docsHref}/querying/querying.html`;
     case 'DOCS_API':
       return `${links.docsHref}/operations/api-reference.html`;
+    case 'DOCS_MSQ_ERROR':
+      return `${links.docsHref}/multi-stage-query/concepts.html#error-codes`;
     case 'COMMUNITY':
       return links.communityHref;
     case 'SLACK':

web-console/src/react-table/react-table-inputs.tsx

@@ -43,7 +43,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend
   const [menuOpen, setMenuOpen] = useState(false);
   const [focused, setFocused] = useState(false);
 
-  const disableComparisons = String(column.headerClassName).includes('disable-comparisons');
+  const enableComparisons = String(column.headerClassName).includes('enable-comparisons');
 
   const { mode, needle } = (filter ? parseFilterModeAndNeedle(filter, true) : undefined) || {
     mode: '~',
@@ -64,7 +64,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend
           onInteraction={setMenuOpen}
           content={
             <Menu>
-              {(disableComparisons ? FILTER_MODES_NO_COMPARISON : FILTER_MODES).map((m, i) => (
+              {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISON).map((m, i) => (
                 <MenuItem
                   key={i}
                   icon={filterModeToIcon(m)}

web-console/src/utils/general.tsx

@@ -40,6 +40,16 @@ export function nonEmptyArray(a: any): a is unknown[] {
   return Array.isArray(a) && Boolean(a.length);
 }
 
+export function isSimpleArray(a: any): a is (string | number | boolean)[] {
+  return (
+    Array.isArray(a) &&
+    a.every(x => {
+      const t = typeof x;
+      return t === 'string' || t === 'number' || t === 'boolean';
+    })
+  );
+}
+
 export function wait(ms: number): Promise<void> {
   return new Promise(resolve => {
     setTimeout(resolve, ms);

web-console/src/views/load-data-view/info-messages.tsx

@@ -16,12 +16,13 @@
  * limitations under the License.
  */
 
-import { Callout, Code, FormGroup } from '@blueprintjs/core';
+import { Button, Callout, Code, FormGroup, Intent } from '@blueprintjs/core';
 import React from 'react';
 
 import { ExternalLink, LearnMore } from '../../components';
 import { DimensionMode, getIngestionDocLink, IngestionSpec } from '../../druid-models';
 import { getLink } from '../../links';
+import { deepGet, deepSet } from '../../utils';
 
 export interface ConnectMessageProps {
   inlineMode: boolean;
@@ -216,3 +217,48 @@ export const SpecMessage = React.memo(function SpecMessage() {
     </FormGroup>
   );
 });
+
+export interface AppendToExistingIssueProps {
+  spec: Partial<IngestionSpec>;
+  onChangeSpec(newSpec: Partial<IngestionSpec>): void;
+}
+
+export const AppendToExistingIssue = React.memo(function AppendToExistingIssue(
+  props: AppendToExistingIssueProps,
+) {
+  const { spec, onChangeSpec } = props;
+
+  const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type');
+  if (
+    partitionsSpecType === 'dynamic' ||
+    deepGet(spec, 'spec.ioConfig.appendToExisting') !== true
+  ) {
+    return null;
+  }
+
+  const dynamicPartitionSpec = {
+    type: 'dynamic',
+    maxRowsPerSegment:
+      deepGet(spec, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment') ||
+      deepGet(spec, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment'),
+  };
+
+  return (
+    <FormGroup>
+      <Callout intent={Intent.DANGER}>
+        <p>
+          Only <Code>dynamic</Code> partitioning supports <Code>appendToExisting: true</Code>. You
+          have currently selected <Code>{partitionsSpecType}</Code> partitioning.
+        </p>
+        <Button
+          intent={Intent.SUCCESS}
+          onClick={() =>
+            onChangeSpec(deepSet(spec, 'spec.tuningConfig.partitionsSpec', dynamicPartitionSpec))
+          }
+        >
+          Change to <Code>dynamic</Code> partitioning
+        </Button>
+      </Callout>
+    </FormGroup>
+  );
+});

web-console/src/views/load-data-view/load-data-view.tsx

@@ -168,6 +168,7 @@ import { ExamplePicker } from './example-picker/example-picker';
 import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table';
 import { FormEditor } from './form-editor/form-editor';
 import {
+  AppendToExistingIssue,
   ConnectMessage,
   FilterMessage,
   ParserMessage,
@@ -1490,7 +1491,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     if (canFlatten && !flattenFields.length && parserQueryState.data) {
       suggestedFlattenFields = computeFlattenPathsForData(
         filterMap(parserQueryState.data.rows, r => r.input),
-        'path',
         'ignore-arrays',
       );
     }
@@ -3003,6 +3003,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <div className="control">
           <PartitionMessage />
           {nonsensicalSingleDimPartitioningMessage}
+          <AppendToExistingIssue spec={spec} onChangeSpec={this.updateSpec} />
         </div>
         {this.renderNextBar({
           disabled: invalidPartitionConfig(spec),
@@ -3096,8 +3097,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 label: 'Append to existing',
                 type: 'boolean',
                 defaultValue: false,
-                defined: spec =>
+                // appendToExisting can only be set on 'dynamic' portioning.
-                  deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') === 'dynamic',
+                // We chose to show it always and instead have a specific message, separate from this form, to notify the user of the issue.
                 info: (
                   <>
                     Creates segments as additional shards of the latest version, effectively
@@ -3166,6 +3167,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         </div>
         <div className="control">
           <PublishMessage />
+          <AppendToExistingIssue spec={spec} onChangeSpec={this.updateSpec} />
         </div>
         {this.renderNextBar({})}
       </>
@@ -3234,6 +3236,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               >{`There is an issue with the spec: ${issueWithSpec}`}</Callout>
             </FormGroup>
           )}
+          <AppendToExistingIssue spec={spec} onChangeSpec={this.updateSpec} />
         </div>
         <div className="next-bar">
           {!isEmptyIngestionSpec(spec) && (

web-console/src/views/query-view/query-input/query-input.tsx

@@ -20,6 +20,7 @@ import { ResizeEntry } from '@blueprintjs/core';
 import { ResizeSensor2 } from '@blueprintjs/popover2';
 import type { Ace } from 'ace-builds';
 import ace from 'ace-builds';
+import { SqlRef, SqlTableRef } from 'druid-query-toolkit';
 import escape from 'lodash.escape';
 import React from 'react';
 import AceEditor from 'react-ace';
@@ -150,7 +151,7 @@ export class QueryInput extends React.PureComponent<QueryInputProps, QueryInputS
     ) {
       const completions = ([] as any[]).concat(
         uniq(columnMetadata.map(d => d.TABLE_SCHEMA)).map(v => ({
-          value: v,
+          value: SqlTableRef.create(v).toString(),
           score: 10,
           meta: 'schema',
         })),
@@ -159,7 +160,7 @@ export class QueryInput extends React.PureComponent<QueryInputProps, QueryInputS
             .filter(d => (currentSchema ? d.TABLE_SCHEMA === currentSchema : true))
             .map(d => d.TABLE_NAME),
         ).map(v => ({
-          value: v,
+          value: SqlTableRef.create(v).toString(),
           score: 49,
           meta: 'datasource',
         })),
@@ -172,7 +173,7 @@ export class QueryInput extends React.PureComponent<QueryInputProps, QueryInputS
             )
             .map(d => d.COLUMN_NAME),
         ).map(v => ({
-          value: v,
+          value: SqlRef.column(v).toString(),
           score: 50,
           meta: 'column',
         })),

web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap

@@ -169,6 +169,7 @@ exports[`SegmentsView matches snapshot 1`] = `
             "accessor": "start",
             "defaultSortDesc": true,
             "filterable": true,
+            "headerClassName": "enable-comparisons",
             "show": true,
             "sortable": true,
             "width": 160,
@@ -179,6 +180,7 @@ exports[`SegmentsView matches snapshot 1`] = `
             "accessor": "end",
             "defaultSortDesc": true,
             "filterable": true,
+            "headerClassName": "enable-comparisons",
             "show": true,
             "sortable": true,
             "width": 160,
@@ -206,7 +208,6 @@ exports[`SegmentsView matches snapshot 1`] = `
             "Cell": [Function],
             "Header": "Shard type",
             "accessor": [Function],
-            "headerClassName": "disable-comparisons",
             "id": "shard_type",
             "show": true,
             "sortable": false,

web-console/src/views/segments-view/segments-view.tsx

@@ -485,7 +485,7 @@ END AS "time_span"`,
     });
   }
 
-  private renderFilterableCell(field: string, disableComparisons = false) {
+  private renderFilterableCell(field: string, enableComparisons = false) {
     const { segmentFilter } = this.state;
 
     return (row: { value: any }) => (
@@ -494,7 +494,7 @@ END AS "time_span"`,
         value={row.value}
         filters={segmentFilter}
         onFiltersChange={filters => this.setState({ segmentFilter: filters })}
-        disableComparisons={disableComparisons}
+        enableComparisons={enableComparisons}
       >
         {row.value}
       </TableFilterableCell>
@@ -582,21 +582,23 @@ END AS "time_span"`,
             Header: 'Start',
             show: visibleColumns.shown('Start'),
             accessor: 'start',
+            headerClassName: 'enable-comparisons',
             width: 160,
             sortable: hasSql,
             defaultSortDesc: true,
             filterable: allowGeneralFilter,
-            Cell: this.renderFilterableCell('start'),
+            Cell: this.renderFilterableCell('start', true),
           },
           {
             Header: 'End',
             show: visibleColumns.shown('End'),
             accessor: 'end',
+            headerClassName: 'enable-comparisons',
             width: 160,
             sortable: hasSql,
             defaultSortDesc: true,
             filterable: allowGeneralFilter,
-            Cell: this.renderFilterableCell('end'),
+            Cell: this.renderFilterableCell('end', true),
           },
           {
             Header: 'Version',
@@ -623,7 +625,6 @@ END AS "time_span"`,
             id: 'shard_type',
             width: 100,
             sortable: false,
-            headerClassName: 'disable-comparisons',
             accessor: d => {
               let v: any;
               try {

web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap

@@ -8,7 +8,12 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `
   <p
     className="error-message-text"
   >
-    TooManyWarnings: 
+    <Memo(ExternalLink)
+      href="https://druid.apache.org/docs/24.0.1/multi-stage-query/concepts.html#error-codes"
+    >
+      TooManyWarnings
+    </Memo(ExternalLink)>
+    : 
     Too many warnings of type CannotParseExternalData generated (max = 10)
   </p>
   <div>

web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx

@@ -20,9 +20,10 @@ import { Callout } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import React, { useState } from 'react';
 
-import { ClickToCopy } from '../../../components';
+import { ClickToCopy, ExternalLink } from '../../../components';
 import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog';
 import { Execution } from '../../../druid-models';
+import { getLink } from '../../../links';
 import { downloadQueryDetailArchive } from '../../../utils';
 
 import './execution-error-pane.scss';
@@ -43,7 +44,12 @@ export const ExecutionErrorPane = React.memo(function ExecutionErrorPane(
   return (
     <Callout className="execution-error-pane" icon={IconNames.ERROR}>
       <p className="error-message-text">
-        {error.errorCode && <>{`${error.errorCode}: `}</>}
+        {error.errorCode && (
+          <>
+            <ExternalLink href={getLink('DOCS_MSQ_ERROR')}>{error.errorCode}</ExternalLink>
+            {': '}
+          </>
+        )}
         {error.errorMessage || (exceptionStackTrace || '').split('\n')[0]}
         {exceptionStackTrace && (
           <>

web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx

@@ -21,6 +21,7 @@ import { ResizeSensor2 } from '@blueprintjs/popover2';
 import type { Ace } from 'ace-builds';
 import ace from 'ace-builds';
 import classNames from 'classnames';
+import { SqlRef, SqlTableRef } from 'druid-query-toolkit';
 import escape from 'lodash.escape';
 import React from 'react';
 import AceEditor from 'react-ace';
@@ -163,7 +164,7 @@ export class FlexibleQueryInput extends React.PureComponent<
     ) {
       const completions = ([] as any[]).concat(
         uniq(columnMetadata.map(d => d.TABLE_SCHEMA)).map(v => ({
-          value: v,
+          value: SqlTableRef.create(v).toString(),
           score: 10,
           meta: 'schema',
         })),
@@ -172,7 +173,7 @@ export class FlexibleQueryInput extends React.PureComponent<
             .filter(d => (currentSchema ? d.TABLE_SCHEMA === currentSchema : true))
             .map(d => d.TABLE_NAME),
         ).map(v => ({
-          value: v,
+          value: SqlTableRef.create(v).toString(),
           score: 49,
           meta: 'datasource',
         })),
@@ -185,7 +186,7 @@ export class FlexibleQueryInput extends React.PureComponent<
             )
             .map(d => d.COLUMN_NAME),
         ).map(v => ({
-          value: v,
+          value: SqlRef.column(v).toString(),
           score: 50,
           meta: 'column',
         })),

web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx

@@ -82,7 +82,7 @@ function jsonValue(ex: SqlExpression, path: string): SqlExpression {
 }
 
 function getJsonPaths(jsons: Record<string, any>[]): string[] {
-  return ['$.'].concat(computeFlattenExprsForData(jsons, 'path', 'include-arrays', true));
+  return ['$.'].concat(computeFlattenExprsForData(jsons, 'include-arrays', true));
 }
 
 function isComparable(x: unknown): boolean {

web-console/unified-console.html

@@ -71,6 +71,6 @@
       };
     </script>
     <script src="console-config.js"></script>
-    <script src="public/web-console-24.0.0.js"></script>
+    <script src="public/web-console-24.0.1.js"></script>
   </body>
 </html>