good stuff (#12435)

web-console/src/blueprint-overrides/_index.scss

@@ -20,6 +20,7 @@
 @import 'common/color-aliases';
 @import 'common/variables';
 @import 'components/button/common';
+@import 'components/button/button-group';
 @import 'components/forms/common';
 @import 'components/navbar/navbar';
 @import 'components/card/card';

web-console/src/blueprint-overrides/components/button/_button-group.scss

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Add body to make the selector more specific than what is in
+// node_modules/@blueprintjs/core/src/components/button/_button-group.scss
+body .#{$ns}-button-group {
+  &:not(.#{$ns}-minimal) {
+    > .#{$ns}-popover-wrapper:not(:last-child) .#{$ns}-button,
+    > .#{$ns}-button:not(:last-child) {
+      // Due to our flat styling this in needed to override the -1px that blueprint tries to set
+      margin-right: 1px;
+    }
+  }
+}

web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap

@@ -168,6 +168,16 @@ exports[`HeaderBar matches snapshot 1`] = `
                 shouldDismissPopover={true}
                 text="Force Overlord mode"
               />
+              <Blueprint4.MenuItem
+                active={false}
+                disabled={false}
+                multiline={false}
+                onClick={[Function]}
+                popoverProps={Object {}}
+                selected={false}
+                shouldDismissPopover={true}
+                text="Force no management proxy mode"
+              />
             </React.Fragment>
           </Blueprint4.MenuItem>
         </Blueprint4.Menu>

web-console/src/components/header-bar/header-bar.tsx

@@ -296,6 +296,12 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
                 onClick={() => setForcedMode(Capabilities.OVERLORD)}
               />
             )}
+            {capabilitiesMode !== 'no-proxy' && (
+              <MenuItem
+                text="Force no management proxy mode"
+                onClick={() => setForcedMode(Capabilities.NO_PROXY)}
+              />
+            )}
           </>
         )}
       </MenuItem>

web-console/src/druid-models/ingestion-spec.spec.ts

@@ -545,8 +545,26 @@ describe('ingestion-spec', () => {
       expect(guessInputFormat(['Obj1lol']).type).toEqual('regex');
     });
 
-    it('works for JSON', () => {
+    it('works for JSON (strict)', () => {
-      expect(guessInputFormat(['{"a":1}']).type).toEqual('json');
+      expect(guessInputFormat(['{"a":1}'])).toEqual({ type: 'json' });
+    });
+
+    it('works for JSON (lax)', () => {
+      expect(guessInputFormat([`{hello:'world'}`])).toEqual({
+        type: 'json',
+        featureSpec: {
+          ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true,
+          ALLOW_COMMENTS: true,
+          ALLOW_MISSING_VALUES: true,
+          ALLOW_NON_NUMERIC_NUMBERS: true,
+          ALLOW_NUMERIC_LEADING_ZEROS: true,
+          ALLOW_SINGLE_QUOTES: true,
+          ALLOW_TRAILING_COMMA: true,
+          ALLOW_UNQUOTED_CONTROL_CHARS: true,
+          ALLOW_UNQUOTED_FIELD_NAMES: true,
+          ALLOW_YAML_COMMENTS: true,
+        },
+      });
     });
 
     it('works for CSV (with header)', () => {

web-console/src/druid-models/ingestion-spec.tsx

@@ -2192,7 +2192,27 @@ export function guessInputFormat(sampleData: string[]): InputFormat {
 
     // If the string starts and ends with curly braces assume JSON
     if (sampleDatum.startsWith('{') && sampleDatum.endsWith('}')) {
-      return inputFormatFromType({ type: 'json' });
+      try {
+        JSON.parse(sampleDatum);
+        return { type: 'json' };
+      } catch {
+        // If the standard JSON parse does not parse then try setting a very lax parsing style
+        return {
+          type: 'json',
+          featureSpec: {
+            ALLOW_COMMENTS: true,
+            ALLOW_YAML_COMMENTS: true,
+            ALLOW_UNQUOTED_FIELD_NAMES: true,
+            ALLOW_SINGLE_QUOTES: true,
+            ALLOW_UNQUOTED_CONTROL_CHARS: true,
+            ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true,
+            ALLOW_NUMERIC_LEADING_ZEROS: true,
+            ALLOW_NON_NUMERIC_NUMBERS: true,
+            ALLOW_MISSING_VALUES: true,
+            ALLOW_TRAILING_COMMA: true,
+          },
+        };
+      }
     }
 
     // Contains more than 3 tabs assume TSV

web-console/src/druid-models/input-format.tsx

@@ -34,7 +34,8 @@ export interface InputFormat {
   readonly listDelimiter?: string;
   readonly pattern?: string;
   readonly function?: string;
-  readonly flattenSpec?: FlattenSpec;
+  readonly flattenSpec?: FlattenSpec | null;
+  readonly featureSpec?: Record<string, boolean>;
   readonly keepNullColumns?: boolean;
 }
 
@@ -58,6 +59,35 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
       </>
     ),
   },
+  {
+    name: 'featureSpec',
+    label: 'JSON parser features',
+    type: 'json',
+    defined: typeIs('json'),
+    info: (
+      <>
+        <p>
+          <ExternalLink href="https://github.com/FasterXML/jackson-core/wiki/JsonParser-Features">
+            JSON parser features
+          </ExternalLink>{' '}
+          supported by Jackson library. Those features will be applied when parsing the input JSON
+          data.
+        </p>
+        <p>
+          Example:{' '}
+          <Code>{`{ "ALLOW_SINGLE_QUOTES": true, "ALLOW_UNQUOTED_FIELD_NAMES": true }`}</Code>
+        </p>
+      </>
+    ),
+  },
+  {
+    name: 'delimiter',
+    type: 'string',
+    defaultValue: '\t',
+    suggestions: ['\t', ';', '|', '#'],
+    defined: typeIs('tsv'),
+    info: <>A custom delimiter for data values.</>,
+  },
   {
     name: 'pattern',
     type: 'string',
@@ -110,14 +140,6 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
       </>
     ),
   },
-  {
-    name: 'delimiter',
-    type: 'string',
-    defaultValue: '\t',
-    suggestions: ['\t', ';', '|', '#'],
-    defined: typeIs('tsv'),
-    info: <>A custom delimiter for data values.</>,
-  },
   {
     name: 'listDelimiter',
     type: 'string',

web-console/src/druid-models/input-source.tsx

@@ -41,6 +41,10 @@ export interface InputSource {
 
   // hdfs
   paths?: string;
+
+  // http
+  httpAuthenticationUsername?: any;
+  httpAuthenticationPassword?: any;
 }
 
 export function issueWithInputSource(inputSource: InputSource | undefined): string | undefined {

web-console/src/utils/capabilities.ts

@@ -45,6 +45,7 @@ export class Capabilities {
   static COORDINATOR_OVERLORD: Capabilities;
   static COORDINATOR: Capabilities;
   static OVERLORD: Capabilities;
+  static NO_PROXY: Capabilities;
 
   private readonly queryType: QueryType;
   private readonly coordinator: boolean;
@@ -96,7 +97,7 @@ export class Capabilities {
 
   static async detectNode(node: 'coordinator' | 'overlord'): Promise<boolean | undefined> {
     try {
-      await Api.instance.get(`/druid/${node === 'overlord' ? 'indexer' : node}/v1/isLeader`, {
+      await Api.instance.get(`/proxy/${node}/status`, {
         timeout: Capabilities.STATUS_TIMEOUT,
       });
     } catch (e) {
@@ -218,3 +219,8 @@ Capabilities.OVERLORD = new Capabilities({
   coordinator: false,
   overlord: true,
 });
+Capabilities.NO_PROXY = new Capabilities({
+  queryType: 'nativeAndSql',
+  coordinator: false,
+  overlord: false,
+});

web-console/src/utils/general.tsx

@@ -310,6 +310,21 @@ export function formatDurationWithMs(ms: NumberLike): string {
   );
 }
 
+export function formatDurationHybrid(ms: NumberLike): string {
+  const n = Number(ms);
+  if (n < 600000) {
+    // anything that looks like 1:23.45 (max 9:59.99)
+    const timeInMin = Math.floor(n / 60000);
+    const timeInSec = Math.floor(n / 1000) % 60;
+    const timeInMs = Math.floor(n) % 1000;
+    return `${timeInMin ? `${timeInMin}:` : ''}${timeInMin ? pad2(timeInSec) : timeInSec}.${pad3(
+      timeInMs,
+    ).substring(0, 2)}s`;
+  } else {
+    return formatDuration(n);
+  }
+}
+
 export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string): string {
   if (!plural) plural = singular + 's';
   return `${formatInteger(n)} ${n === 1 ? singular : plural}`;
@@ -512,3 +527,7 @@ export function hashJoaat(str: string): number {
 export function objectHash(obj: any): string {
   return hashJoaat(JSONBig.stringify(obj)).toString(16).padStart(8);
 }
+
+export function hasPopoverOpen(): boolean {
+  return Boolean(document.querySelector('.bp4-portal .bp4-overlay .bp4-popover2'));
+}

web-console/src/utils/sampler.ts

@@ -139,13 +139,15 @@ export interface HeaderFromSampleResponseOptions {
   ignoreTimeColumn?: boolean;
   columnOrder?: string[];
   suffixColumnOrder?: string[];
+  useInput?: boolean;
 }
 
 export function headerFromSampleResponse(options: HeaderFromSampleResponseOptions): string[] {
-  const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder } = options;
+  const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder, useInput } = options;
 
+  const key = useInput ? 'input' : 'parsed';
   let columns = arrangeWithPrefixSuffix(
-    dedupe(sampleResponse.data.flatMap(s => (s.parsed ? Object.keys(s.parsed) : []))),
+    dedupe(sampleResponse.data.flatMap(s => (s[key] ? Object.keys(s[key]!) : []))),
     columnOrder || [TIME_COLUMN],
     suffixColumnOrder || [],
   );

web-console/src/variables.scss

@@ -35,7 +35,7 @@ $druid-brand-background: #1c1c26;
   background: $white;
   border-radius: $pt-border-radius;
 
-  .bp3-dark & {
+  .#{$bp-ns}-dark & {
     background: $dark-gray3;
   }
 }

web-console/src/views/datasource-view/datasource-view.tsx

@@ -57,6 +57,7 @@ import {
   formatMillions,
   formatPercent,
   getDruidErrorMessage,
+  hasPopoverOpen,
   isNumberLikeNaN,
   LocalStorageBackedVisibility,
   LocalStorageKeys,
@@ -493,7 +494,8 @@ ORDER BY 1`;
     });
   }
 
-  private readonly refresh = (auto: any): void => {
+  private readonly refresh = (auto: boolean): void => {
+    if (auto && hasPopoverOpen()) return;
     this.datasourceQueryManager.rerunLastQuery(auto);
     this.tiersQueryManager.rerunLastQuery(auto);
   };

web-console/src/views/ingestion-view/ingestion-view.tsx

@@ -47,6 +47,7 @@ import {
   deepGet,
   formatDuration,
   getDruidErrorMessage,
+  hasPopoverOpen,
   LocalStorageBackedVisibility,
   localStorageGet,
   LocalStorageKeys,
@@ -1089,7 +1090,10 @@ ORDER BY "rank" DESC, "created_time" DESC`;
             <ViewControlBar label="Supervisors">
               <RefreshButton
                 localStorageKey={LocalStorageKeys.SUPERVISORS_REFRESH_RATE}
-                onRefresh={auto => this.supervisorQueryManager.rerunLastQuery(auto)}
+                onRefresh={auto => {
+                  if (auto && hasPopoverOpen()) return;
+                  this.supervisorQueryManager.rerunLastQuery(auto);
+                }}
               />
               {this.renderBulkSupervisorActions()}
               <TableColumnSelector
@@ -1141,7 +1145,10 @@ ORDER BY "rank" DESC, "created_time" DESC`;
               </ButtonGroup>
               <RefreshButton
                 localStorageKey={LocalStorageKeys.TASKS_REFRESH_RATE}
-                onRefresh={auto => this.taskQueryManager.rerunLastQuery(auto)}
+                onRefresh={auto => {
+                  if (auto && hasPopoverOpen()) return;
+                  this.taskQueryManager.rerunLastQuery(auto);
+                }}
               />
               {this.renderBulkTasksActions()}
               <TableColumnSelector

web-console/src/views/load-data-view/load-data-view.tsx

@@ -1463,7 +1463,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               )}
             </>
           )}
-          {this.renderFlattenControls()}
+          {canFlatten && this.renderFlattenControls()}
           {suggestedFlattenFields && suggestedFlattenFields.length ? (
             <FormGroup>
               <Button

web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx

@@ -41,6 +41,7 @@ export interface ParseDataTableProps {
   flattenedColumnsOnly: boolean;
   flattenFields: FlattenField[];
   onFlattenFieldSelect: (field: FlattenField, index: number) => void;
+  useInput?: boolean;
 }
 
 export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDataTableProps) {
@@ -51,8 +52,10 @@ export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDat
     flattenedColumnsOnly,
     flattenFields,
     onFlattenFieldSelect,
+    useInput,
   } = props;
 
+  const key = useInput ? 'input' : 'parsed';
   return (
     <ReactTable
       className="parse-data-table -striped -highlight"
@@ -82,7 +85,7 @@ export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDat
             </div>
           ),
           id: String(i),
-          accessor: (row: SampleEntry) => (row.parsed ? row.parsed[columnName] : null),
+          accessor: (row: SampleEntry) => (row[key] ? row[key]![columnName] : null),
           Cell: function ParseDataTableCell(row) {
             if (row.original.unparseable) {
               return <TableCellUnparseable />;

web-console/src/views/lookups-view/lookups-view.tsx

@@ -37,6 +37,7 @@ import { Api, AppToaster } from '../../singletons';
 import {
   deepGet,
   getDruidErrorMessage,
+  hasPopoverOpen,
   isLookupsUninitialized,
   LocalStorageBackedVisibility,
   LocalStorageKeys,
@@ -455,7 +456,10 @@ export class LookupsView extends React.PureComponent<LookupsViewProps, LookupsVi
       <div className="lookups-view app-view">
         <ViewControlBar label="Lookups">
           <RefreshButton
-            onRefresh={auto => this.lookupsQueryManager.rerunLastQuery(auto)}
+            onRefresh={auto => {
+              if (auto && hasPopoverOpen()) return;
+              this.lookupsQueryManager.rerunLastQuery(auto);
+            }}
             localStorageKey={LocalStorageKeys.LOOKUPS_REFRESH_RATE}
           />
           {!lookupEntriesAndTiersState.isError() && (

web-console/src/views/query-view/query-timer/query-timer.tsx

@@ -21,6 +21,7 @@ import { IconNames } from '@blueprintjs/icons';
 import React, { useState } from 'react';
 
 import { useInterval } from '../../../hooks';
+import { formatDurationHybrid } from '../../../utils';
 
 import './query-timer.scss';
 
@@ -37,9 +38,10 @@ export const QueryTimer = React.memo(function QueryTimer() {
 
   const elapsed = currentTime - startTime;
   if (elapsed <= 0) return null;
+
   return (
     <div className="query-timer">
-      {`${(elapsed / 1000).toFixed(2)}s`}
+      {formatDurationHybrid(elapsed)}
       <Button icon={IconNames.STOPWATCH} minimal />
     </div>
   );

web-console/src/views/query-view/query-view.tsx

@@ -201,7 +201,9 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
       },
     });
 
-    const queryRunner = new QueryRunner();
+    const queryRunner = new QueryRunner({
+      inflateDateStrategy: 'none',
+    });
 
     this.queryManager = new QueryManager({
       processQuery: async (

web-console/src/views/segments-view/segments-view.tsx

@@ -51,6 +51,7 @@ import {
   formatBytes,
   formatInteger,
   getNeedleAndMode,
+  hasPopoverOpen,
   isNumberLikeNaN,
   LocalStorageBackedVisibility,
   LocalStorageKeys,
@@ -893,7 +894,10 @@ END AS "time_span"`,
         >
           <ViewControlBar label="Segments">
             <RefreshButton
-              onRefresh={auto => this.segmentsQueryManager.rerunLastQuery(auto)}
+              onRefresh={auto => {
+                if (auto && hasPopoverOpen()) return;
+                this.segmentsQueryManager.rerunLastQuery(auto);
+              }}
               localStorageKey={LocalStorageKeys.SEGMENTS_REFRESH_RATE}
             />
             <Label>Group by</Label>

web-console/src/views/services-view/services-view.tsx

@@ -41,6 +41,7 @@ import {
   deepGet,
   formatBytes,
   formatBytesCompact,
+  hasPopoverOpen,
   LocalStorageBackedVisibility,
   LocalStorageKeys,
   lookupBy,
@@ -718,7 +719,10 @@ ORDER BY
             </Button>
           </ButtonGroup>
           <RefreshButton
-            onRefresh={auto => this.serviceQueryManager.rerunLastQuery(auto)}
+            onRefresh={auto => {
+              if (auto && hasPopoverOpen()) return;
+              this.serviceQueryManager.rerunLastQuery(auto);
+            }}
             localStorageKey={LocalStorageKeys.SERVICES_REFRESH_RATE}
           />
           {this.renderBulkServicesActions()}