good stuff (#12435)

2022-04-14 00:23:06 -07:00 · 2022-04-14 00:23:06 -07:00 · a72cc28959
parent 5824ab9608
commit a72cc28959
21 changed files with 191 additions and 26 deletions
--- a/web-console/src/blueprint-overrides/_index.scss
+++ b/web-console/src/blueprint-overrides/_index.scss
@ -20,6 +20,7 @@
@import 'common/color-aliases';
@import 'common/variables';
@import 'components/button/common';
+@import 'components/button/button-group';
@import 'components/forms/common';
@import 'components/navbar/navbar';
@import 'components/card/card';
--- a/web-console/src/blueprint-overrides/components/button/_button-group.scss
+++ b/web-console/src/blueprint-overrides/components/button/_button-group.scss
@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Add body to make the selector more specific than what is in
+// node_modules/@blueprintjs/core/src/components/button/_button-group.scss
+body .#{$ns}-button-group {
+  &:not(.#{$ns}-minimal) {
+    > .#{$ns}-popover-wrapper:not(:last-child) .#{$ns}-button,
+    > .#{$ns}-button:not(:last-child) {
+      // Due to our flat styling this in needed to override the -1px that blueprint tries to set
+      margin-right: 1px;
+    }
+  }
+}
--- a/web-console/src/components/header-bar/snapshots/header-bar.spec.tsx.snap
+++ b/web-console/src/components/header-bar/snapshots/header-bar.spec.tsx.snap
@ -168,6 +168,16 @@ exports[`HeaderBar matches snapshot 1`] = `
                shouldDismissPopover={true}
                text="Force Overlord mode"
              />
+              <Blueprint4.MenuItem
+                active={false}
+                disabled={false}
+                multiline={false}
+                onClick={[Function]}
+                popoverProps={Object {}}
+                selected={false}
+                shouldDismissPopover={true}
+                text="Force no management proxy mode"
+              />
            </React.Fragment>
          </Blueprint4.MenuItem>
        </Blueprint4.Menu>
--- a/web-console/src/components/header-bar/header-bar.tsx
+++ b/web-console/src/components/header-bar/header-bar.tsx
@ -296,6 +296,12 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
                onClick={() => setForcedMode(Capabilities.OVERLORD)}
              />
            )}
+            {capabilitiesMode !== 'no-proxy' && (
+              <MenuItem
+                text="Force no management proxy mode"
+                onClick={() => setForcedMode(Capabilities.NO_PROXY)}
+              />
+            )}
          </>
        )}
      </MenuItem>
--- a/web-console/src/druid-models/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec.spec.ts
@ -545,8 +545,26 @@ describe('ingestion-spec', () => {
      expect(guessInputFormat(['Obj1lol']).type).toEqual('regex');
    });

-    it('works for JSON', () => {
-      expect(guessInputFormat(['{"a":1}']).type).toEqual('json');
+    it('works for JSON (strict)', () => {
+      expect(guessInputFormat(['{"a":1}'])).toEqual({ type: 'json' });
+    });
+
+    it('works for JSON (lax)', () => {
+      expect(guessInputFormat([`{hello:'world'}`])).toEqual({
+        type: 'json',
+        featureSpec: {
+          ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true,
+          ALLOW_COMMENTS: true,
+          ALLOW_MISSING_VALUES: true,
+          ALLOW_NON_NUMERIC_NUMBERS: true,
+          ALLOW_NUMERIC_LEADING_ZEROS: true,
+          ALLOW_SINGLE_QUOTES: true,
+          ALLOW_TRAILING_COMMA: true,
+          ALLOW_UNQUOTED_CONTROL_CHARS: true,
+          ALLOW_UNQUOTED_FIELD_NAMES: true,
+          ALLOW_YAML_COMMENTS: true,
+        },
+      });
    });

    it('works for CSV (with header)', () => {
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@ -2192,7 +2192,27 @@ export function guessInputFormat(sampleData: string[]): InputFormat {

    // If the string starts and ends with curly braces assume JSON
    if (sampleDatum.startsWith('{') && sampleDatum.endsWith('}')) {
-      return inputFormatFromType({ type: 'json' });
+      try {
+        JSON.parse(sampleDatum);
+        return { type: 'json' };
+      } catch {
+        // If the standard JSON parse does not parse then try setting a very lax parsing style
+        return {
+          type: 'json',
+          featureSpec: {
+            ALLOW_COMMENTS: true,
+            ALLOW_YAML_COMMENTS: true,
+            ALLOW_UNQUOTED_FIELD_NAMES: true,
+            ALLOW_SINGLE_QUOTES: true,
+            ALLOW_UNQUOTED_CONTROL_CHARS: true,
+            ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true,
+            ALLOW_NUMERIC_LEADING_ZEROS: true,
+            ALLOW_NON_NUMERIC_NUMBERS: true,
+            ALLOW_MISSING_VALUES: true,
+            ALLOW_TRAILING_COMMA: true,
+          },
+        };
+      }
    }

    // Contains more than 3 tabs assume TSV
--- a/web-console/src/druid-models/input-format.tsx
+++ b/web-console/src/druid-models/input-format.tsx
@ -34,7 +34,8 @@ export interface InputFormat {
  readonly listDelimiter?: string;
  readonly pattern?: string;
  readonly function?: string;
-  readonly flattenSpec?: FlattenSpec;
+  readonly flattenSpec?: FlattenSpec | null;
+  readonly featureSpec?: Record<string, boolean>;
  readonly keepNullColumns?: boolean;
 }

@ -58,6 +59,35 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
      </>
    ),
  },
+  {
+    name: 'featureSpec',
+    label: 'JSON parser features',
+    type: 'json',
+    defined: typeIs('json'),
+    info: (
+      <>
+        <p>
+          <ExternalLink href="https://github.com/FasterXML/jackson-core/wiki/JsonParser-Features">
+            JSON parser features
+          </ExternalLink>{' '}
+          supported by Jackson library. Those features will be applied when parsing the input JSON
+          data.
+        </p>
+        <p>
+          Example:{' '}
+          <Code>{`{ "ALLOW_SINGLE_QUOTES": true, "ALLOW_UNQUOTED_FIELD_NAMES": true }`}</Code>
+        </p>
+      </>
+    ),
+  },
+  {
+    name: 'delimiter',
+    type: 'string',
+    defaultValue: '\t',
+    suggestions: ['\t', ';', '|', '#'],
+    defined: typeIs('tsv'),
+    info: <>A custom delimiter for data values.</>,
+  },
  {
    name: 'pattern',
    type: 'string',
@ -110,14 +140,6 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
      </>
    ),
  },
-  {
-    name: 'delimiter',
-    type: 'string',
-    defaultValue: '\t',
-    suggestions: ['\t', ';', '|', '#'],
-    defined: typeIs('tsv'),
-    info: <>A custom delimiter for data values.</>,
-  },
  {
    name: 'listDelimiter',
    type: 'string',
--- a/web-console/src/druid-models/input-source.tsx
+++ b/web-console/src/druid-models/input-source.tsx
@ -41,6 +41,10 @@ export interface InputSource {

  // hdfs
  paths?: string;
+
+  // http
+  httpAuthenticationUsername?: any;
+  httpAuthenticationPassword?: any;
 }

 export function issueWithInputSource(inputSource: InputSource | undefined): string | undefined {
--- a/web-console/src/utils/capabilities.ts
+++ b/web-console/src/utils/capabilities.ts
@ -45,6 +45,7 @@ export class Capabilities {
  static COORDINATOR_OVERLORD: Capabilities;
  static COORDINATOR: Capabilities;
  static OVERLORD: Capabilities;
+  static NO_PROXY: Capabilities;

  private readonly queryType: QueryType;
  private readonly coordinator: boolean;
@ -96,7 +97,7 @@ export class Capabilities {

  static async detectNode(node: 'coordinator' | 'overlord'): Promise<boolean | undefined> {
    try {
-      await Api.instance.get(`/druid/${node === 'overlord' ? 'indexer' : node}/v1/isLeader`, {
+      await Api.instance.get(`/proxy/${node}/status`, {
        timeout: Capabilities.STATUS_TIMEOUT,
      });
    } catch (e) {
@ -218,3 +219,8 @@ Capabilities.OVERLORD = new Capabilities({
  coordinator: false,
  overlord: true,
 });
+Capabilities.NO_PROXY = new Capabilities({
+  queryType: 'nativeAndSql',
+  coordinator: false,
+  overlord: false,
+});
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@ -310,6 +310,21 @@ export function formatDurationWithMs(ms: NumberLike): string {
  );
 }

+export function formatDurationHybrid(ms: NumberLike): string {
+  const n = Number(ms);
+  if (n < 600000) {
+    // anything that looks like 1:23.45 (max 9:59.99)
+    const timeInMin = Math.floor(n / 60000);
+    const timeInSec = Math.floor(n / 1000) % 60;
+    const timeInMs = Math.floor(n) % 1000;
+    return `${timeInMin ? `${timeInMin}:` : ''}${timeInMin ? pad2(timeInSec) : timeInSec}.${pad3(
+      timeInMs,
+    ).substring(0, 2)}s`;
+  } else {
+    return formatDuration(n);
+  }
+}
+
 export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string): string {
  if (!plural) plural = singular + 's';
  return `${formatInteger(n)} ${n === 1 ? singular : plural}`;
@ -512,3 +527,7 @@ export function hashJoaat(str: string): number {
 export function objectHash(obj: any): string {
  return hashJoaat(JSONBig.stringify(obj)).toString(16).padStart(8);
 }
+
+export function hasPopoverOpen(): boolean {
+  return Boolean(document.querySelector('.bp4-portal .bp4-overlay .bp4-popover2'));
+}
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@ -139,13 +139,15 @@ export interface HeaderFromSampleResponseOptions {
  ignoreTimeColumn?: boolean;
  columnOrder?: string[];
  suffixColumnOrder?: string[];
+  useInput?: boolean;
 }

 export function headerFromSampleResponse(options: HeaderFromSampleResponseOptions): string[] {
-  const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder } = options;
+  const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder, useInput } = options;

+  const key = useInput ? 'input' : 'parsed';
  let columns = arrangeWithPrefixSuffix(
-    dedupe(sampleResponse.data.flatMap(s => (s.parsed ? Object.keys(s.parsed) : []))),
+    dedupe(sampleResponse.data.flatMap(s => (s[key] ? Object.keys(s[key]!) : []))),
    columnOrder || [TIME_COLUMN],
    suffixColumnOrder || [],
  );
--- a/web-console/src/variables.scss
+++ b/web-console/src/variables.scss
@ -35,7 +35,7 @@ $druid-brand-background: #1c1c26;
  background: $white;
  border-radius: $pt-border-radius;

-  .bp3-dark & {
+  .#{$bp-ns}-dark & {
    background: $dark-gray3;
  }
 }
--- a/web-console/src/views/datasource-view/datasource-view.tsx
+++ b/web-console/src/views/datasource-view/datasource-view.tsx
@ -57,6 +57,7 @@ import {
  formatMillions,
  formatPercent,
  getDruidErrorMessage,
+  hasPopoverOpen,
  isNumberLikeNaN,
  LocalStorageBackedVisibility,
  LocalStorageKeys,
@ -493,7 +494,8 @@ ORDER BY 1`;
    });
  }

-  private readonly refresh = (auto: any): void => {
+  private readonly refresh = (auto: boolean): void => {
+    if (auto && hasPopoverOpen()) return;
    this.datasourceQueryManager.rerunLastQuery(auto);
    this.tiersQueryManager.rerunLastQuery(auto);
  };
--- a/web-console/src/views/ingestion-view/ingestion-view.tsx
+++ b/web-console/src/views/ingestion-view/ingestion-view.tsx
@ -47,6 +47,7 @@ import {
  deepGet,
  formatDuration,
  getDruidErrorMessage,
+  hasPopoverOpen,
  LocalStorageBackedVisibility,
  localStorageGet,
  LocalStorageKeys,
@ -1089,7 +1090,10 @@ ORDER BY "rank" DESC, "created_time" DESC`;
            <ViewControlBar label="Supervisors">
              <RefreshButton
                localStorageKey={LocalStorageKeys.SUPERVISORS_REFRESH_RATE}
-                onRefresh={auto => this.supervisorQueryManager.rerunLastQuery(auto)}
+                onRefresh={auto => {
+                  if (auto && hasPopoverOpen()) return;
+                  this.supervisorQueryManager.rerunLastQuery(auto);
+                }}
              />
              {this.renderBulkSupervisorActions()}
              <TableColumnSelector
@ -1141,7 +1145,10 @@ ORDER BY "rank" DESC, "created_time" DESC`;
              </ButtonGroup>
              <RefreshButton
                localStorageKey={LocalStorageKeys.TASKS_REFRESH_RATE}
-                onRefresh={auto => this.taskQueryManager.rerunLastQuery(auto)}
+                onRefresh={auto => {
+                  if (auto && hasPopoverOpen()) return;
+                  this.taskQueryManager.rerunLastQuery(auto);
+                }}
              />
              {this.renderBulkTasksActions()}
              <TableColumnSelector
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@ -1463,7 +1463,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
              )}
            </>
          )}
-          {this.renderFlattenControls()}
+          {canFlatten && this.renderFlattenControls()}
          {suggestedFlattenFields && suggestedFlattenFields.length ? (
            <FormGroup>
              <Button
--- a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
+++ b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
@ -41,6 +41,7 @@ export interface ParseDataTableProps {
  flattenedColumnsOnly: boolean;
  flattenFields: FlattenField[];
  onFlattenFieldSelect: (field: FlattenField, index: number) => void;
+  useInput?: boolean;
 }

 export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDataTableProps) {
@ -51,8 +52,10 @@ export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDat
    flattenedColumnsOnly,
    flattenFields,
    onFlattenFieldSelect,
+    useInput,
  } = props;

+  const key = useInput ? 'input' : 'parsed';
  return (
    <ReactTable
      className="parse-data-table -striped -highlight"
@ -82,7 +85,7 @@ export const ParseDataTable = React.memo(function ParseDataTable(props: ParseDat
            </div>
          ),
          id: String(i),
-          accessor: (row: SampleEntry) => (row.parsed ? row.parsed[columnName] : null),
+          accessor: (row: SampleEntry) => (row[key] ? row[key]![columnName] : null),
          Cell: function ParseDataTableCell(row) {
            if (row.original.unparseable) {
              return <TableCellUnparseable />;
--- a/web-console/src/views/lookups-view/lookups-view.tsx
+++ b/web-console/src/views/lookups-view/lookups-view.tsx
@ -37,6 +37,7 @@ import { Api, AppToaster } from '../../singletons';
 import {
  deepGet,
  getDruidErrorMessage,
+  hasPopoverOpen,
  isLookupsUninitialized,
  LocalStorageBackedVisibility,
  LocalStorageKeys,
@ -455,7 +456,10 @@ export class LookupsView extends React.PureComponent<LookupsViewProps, LookupsVi
      <div className="lookups-view app-view">
        <ViewControlBar label="Lookups">
          <RefreshButton
-            onRefresh={auto => this.lookupsQueryManager.rerunLastQuery(auto)}
+            onRefresh={auto => {
+              if (auto && hasPopoverOpen()) return;
+              this.lookupsQueryManager.rerunLastQuery(auto);
+            }}
            localStorageKey={LocalStorageKeys.LOOKUPS_REFRESH_RATE}
          />
          {!lookupEntriesAndTiersState.isError() && (
--- a/web-console/src/views/query-view/query-timer/query-timer.tsx
+++ b/web-console/src/views/query-view/query-timer/query-timer.tsx
@ -21,6 +21,7 @@ import { IconNames } from '@blueprintjs/icons';
 import React, { useState } from 'react';

 import { useInterval } from '../../../hooks';
+import { formatDurationHybrid } from '../../../utils';

 import './query-timer.scss';

@ -37,9 +38,10 @@ export const QueryTimer = React.memo(function QueryTimer() {

  const elapsed = currentTime - startTime;
  if (elapsed <= 0) return null;
+
  return (
    <div className="query-timer">
-      {`${(elapsed / 1000).toFixed(2)}s`}
+      {formatDurationHybrid(elapsed)}
      <Button icon={IconNames.STOPWATCH} minimal />
    </div>
  );
--- a/web-console/src/views/query-view/query-view.tsx
+++ b/web-console/src/views/query-view/query-view.tsx
@ -201,7 +201,9 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
      },
    });

-    const queryRunner = new QueryRunner();
+    const queryRunner = new QueryRunner({
+      inflateDateStrategy: 'none',
+    });

    this.queryManager = new QueryManager({
      processQuery: async (
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@ -51,6 +51,7 @@ import {
  formatBytes,
  formatInteger,
  getNeedleAndMode,
+  hasPopoverOpen,
  isNumberLikeNaN,
  LocalStorageBackedVisibility,
  LocalStorageKeys,
@ -893,7 +894,10 @@ END AS "time_span"`,
        >
          <ViewControlBar label="Segments">
            <RefreshButton
-              onRefresh={auto => this.segmentsQueryManager.rerunLastQuery(auto)}
+              onRefresh={auto => {
+                if (auto && hasPopoverOpen()) return;
+                this.segmentsQueryManager.rerunLastQuery(auto);
+              }}
              localStorageKey={LocalStorageKeys.SEGMENTS_REFRESH_RATE}
            />
            <Label>Group by</Label>
--- a/web-console/src/views/services-view/services-view.tsx
+++ b/web-console/src/views/services-view/services-view.tsx
@ -41,6 +41,7 @@ import {
  deepGet,
  formatBytes,
  formatBytesCompact,
+  hasPopoverOpen,
  LocalStorageBackedVisibility,
  LocalStorageKeys,
  lookupBy,
@ -718,7 +719,10 @@ ORDER BY
            </Button>
          </ButtonGroup>
          <RefreshButton
-            onRefresh={auto => this.serviceQueryManager.rerunLastQuery(auto)}
+            onRefresh={auto => {
+              if (auto && hasPopoverOpen()) return;
+              this.serviceQueryManager.rerunLastQuery(auto);
+            }}
            localStorageKey={LocalStorageKeys.SERVICES_REFRESH_RATE}
          />
          {this.renderBulkServicesActions()}