diff --git a/web-console/src/blueprint-overrides/_index.scss b/web-console/src/blueprint-overrides/_index.scss index 821d8a395d6..faffe22bbb9 100644 --- a/web-console/src/blueprint-overrides/_index.scss +++ b/web-console/src/blueprint-overrides/_index.scss @@ -20,6 +20,7 @@ @import 'common/color-aliases'; @import 'common/variables'; @import 'components/button/common'; +@import 'components/button/button-group'; @import 'components/forms/common'; @import 'components/navbar/navbar'; @import 'components/card/card'; diff --git a/web-console/src/blueprint-overrides/components/button/_button-group.scss b/web-console/src/blueprint-overrides/components/button/_button-group.scss new file mode 100644 index 00000000000..9564e8d6117 --- /dev/null +++ b/web-console/src/blueprint-overrides/components/button/_button-group.scss @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Add body to make the selector more specific than what is in +// node_modules/@blueprintjs/core/src/components/button/_button-group.scss +body .#{$ns}-button-group { + &:not(.#{$ns}-minimal) { + > .#{$ns}-popover-wrapper:not(:last-child) .#{$ns}-button, + > .#{$ns}-button:not(:last-child) { + // Due to our flat styling this in needed to override the -1px that blueprint tries to set + margin-right: 1px; + } + } +} diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index f927f27c0b8..bfd6aa4c3da 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -168,6 +168,16 @@ exports[`HeaderBar matches snapshot 1`] = ` shouldDismissPopover={true} text="Force Overlord mode" /> + diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx index 390b49e5afe..217ff807571 100644 --- a/web-console/src/components/header-bar/header-bar.tsx +++ b/web-console/src/components/header-bar/header-bar.tsx @@ -296,6 +296,12 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) { onClick={() => setForcedMode(Capabilities.OVERLORD)} /> )} + {capabilitiesMode !== 'no-proxy' && ( + setForcedMode(Capabilities.NO_PROXY)} + /> + )} )} diff --git a/web-console/src/druid-models/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec.spec.ts index 52efaed3a82..768a061fd93 100644 --- a/web-console/src/druid-models/ingestion-spec.spec.ts +++ b/web-console/src/druid-models/ingestion-spec.spec.ts @@ -545,8 +545,26 @@ describe('ingestion-spec', () => { expect(guessInputFormat(['Obj1lol']).type).toEqual('regex'); }); - it('works for JSON', () => { - expect(guessInputFormat(['{"a":1}']).type).toEqual('json'); + it('works for JSON (strict)', () => { + expect(guessInputFormat(['{"a":1}'])).toEqual({ type: 'json' }); + }); + + it('works for JSON (lax)', () => { + expect(guessInputFormat([`{hello:'world'}`])).toEqual({ + type: 'json', + featureSpec: { + ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true, + ALLOW_COMMENTS: true, + ALLOW_MISSING_VALUES: true, + ALLOW_NON_NUMERIC_NUMBERS: true, + ALLOW_NUMERIC_LEADING_ZEROS: true, + ALLOW_SINGLE_QUOTES: true, + ALLOW_TRAILING_COMMA: true, + ALLOW_UNQUOTED_CONTROL_CHARS: true, + ALLOW_UNQUOTED_FIELD_NAMES: true, + ALLOW_YAML_COMMENTS: true, + }, + }); }); it('works for CSV (with header)', () => { diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx index e58aeef3207..fd7a92a1b4b 100644 --- a/web-console/src/druid-models/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec.tsx @@ -2192,7 +2192,27 @@ export function guessInputFormat(sampleData: string[]): InputFormat { // If the string starts and ends with curly braces assume JSON if (sampleDatum.startsWith('{') && sampleDatum.endsWith('}')) { - return inputFormatFromType({ type: 'json' }); + try { + JSON.parse(sampleDatum); + return { type: 'json' }; + } catch { + // If the standard JSON parse does not parse then try setting a very lax parsing style + return { + type: 'json', + featureSpec: { + ALLOW_COMMENTS: true, + ALLOW_YAML_COMMENTS: true, + ALLOW_UNQUOTED_FIELD_NAMES: true, + ALLOW_SINGLE_QUOTES: true, + ALLOW_UNQUOTED_CONTROL_CHARS: true, + ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER: true, + ALLOW_NUMERIC_LEADING_ZEROS: true, + ALLOW_NON_NUMERIC_NUMBERS: true, + ALLOW_MISSING_VALUES: true, + ALLOW_TRAILING_COMMA: true, + }, + }; + } } // Contains more than 3 tabs assume TSV diff --git a/web-console/src/druid-models/input-format.tsx b/web-console/src/druid-models/input-format.tsx index f781a21f4cd..90fb8b800d1 100644 --- a/web-console/src/druid-models/input-format.tsx +++ b/web-console/src/druid-models/input-format.tsx @@ -34,7 +34,8 @@ export interface InputFormat { readonly listDelimiter?: string; readonly pattern?: string; readonly function?: string; - readonly flattenSpec?: FlattenSpec; + readonly flattenSpec?: FlattenSpec | null; + readonly featureSpec?: Record; readonly keepNullColumns?: boolean; } @@ -58,6 +59,35 @@ export const INPUT_FORMAT_FIELDS: Field[] = [ ), }, + { + name: 'featureSpec', + label: 'JSON parser features', + type: 'json', + defined: typeIs('json'), + info: ( + <> +

+ + JSON parser features + {' '} + supported by Jackson library. Those features will be applied when parsing the input JSON + data. +

+

+ Example:{' '} + {`{ "ALLOW_SINGLE_QUOTES": true, "ALLOW_UNQUOTED_FIELD_NAMES": true }`} +

+ + ), + }, + { + name: 'delimiter', + type: 'string', + defaultValue: '\t', + suggestions: ['\t', ';', '|', '#'], + defined: typeIs('tsv'), + info: <>A custom delimiter for data values., + }, { name: 'pattern', type: 'string', @@ -110,14 +140,6 @@ export const INPUT_FORMAT_FIELDS: Field[] = [ ), }, - { - name: 'delimiter', - type: 'string', - defaultValue: '\t', - suggestions: ['\t', ';', '|', '#'], - defined: typeIs('tsv'), - info: <>A custom delimiter for data values., - }, { name: 'listDelimiter', type: 'string', diff --git a/web-console/src/druid-models/input-source.tsx b/web-console/src/druid-models/input-source.tsx index 8c4302e28b2..116ef48a839 100644 --- a/web-console/src/druid-models/input-source.tsx +++ b/web-console/src/druid-models/input-source.tsx @@ -41,6 +41,10 @@ export interface InputSource { // hdfs paths?: string; + + // http + httpAuthenticationUsername?: any; + httpAuthenticationPassword?: any; } export function issueWithInputSource(inputSource: InputSource | undefined): string | undefined { diff --git a/web-console/src/utils/capabilities.ts b/web-console/src/utils/capabilities.ts index 4ad0dd2fb6b..406115dfc99 100644 --- a/web-console/src/utils/capabilities.ts +++ b/web-console/src/utils/capabilities.ts @@ -45,6 +45,7 @@ export class Capabilities { static COORDINATOR_OVERLORD: Capabilities; static COORDINATOR: Capabilities; static OVERLORD: Capabilities; + static NO_PROXY: Capabilities; private readonly queryType: QueryType; private readonly coordinator: boolean; @@ -96,7 +97,7 @@ export class Capabilities { static async detectNode(node: 'coordinator' | 'overlord'): Promise { try { - await Api.instance.get(`/druid/${node === 'overlord' ? 'indexer' : node}/v1/isLeader`, { + await Api.instance.get(`/proxy/${node}/status`, { timeout: Capabilities.STATUS_TIMEOUT, }); } catch (e) { @@ -218,3 +219,8 @@ Capabilities.OVERLORD = new Capabilities({ coordinator: false, overlord: true, }); +Capabilities.NO_PROXY = new Capabilities({ + queryType: 'nativeAndSql', + coordinator: false, + overlord: false, +}); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index fb651e18296..e1c064a0059 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -310,6 +310,21 @@ export function formatDurationWithMs(ms: NumberLike): string { ); } +export function formatDurationHybrid(ms: NumberLike): string { + const n = Number(ms); + if (n < 600000) { + // anything that looks like 1:23.45 (max 9:59.99) + const timeInMin = Math.floor(n / 60000); + const timeInSec = Math.floor(n / 1000) % 60; + const timeInMs = Math.floor(n) % 1000; + return `${timeInMin ? `${timeInMin}:` : ''}${timeInMin ? pad2(timeInSec) : timeInSec}.${pad3( + timeInMs, + ).substring(0, 2)}s`; + } else { + return formatDuration(n); + } +} + export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string): string { if (!plural) plural = singular + 's'; return `${formatInteger(n)} ${n === 1 ? singular : plural}`; @@ -512,3 +527,7 @@ export function hashJoaat(str: string): number { export function objectHash(obj: any): string { return hashJoaat(JSONBig.stringify(obj)).toString(16).padStart(8); } + +export function hasPopoverOpen(): boolean { + return Boolean(document.querySelector('.bp4-portal .bp4-overlay .bp4-popover2')); +} diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index a8619a5bafd..5ebd49d67b1 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -139,13 +139,15 @@ export interface HeaderFromSampleResponseOptions { ignoreTimeColumn?: boolean; columnOrder?: string[]; suffixColumnOrder?: string[]; + useInput?: boolean; } export function headerFromSampleResponse(options: HeaderFromSampleResponseOptions): string[] { - const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder } = options; + const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder, useInput } = options; + const key = useInput ? 'input' : 'parsed'; let columns = arrangeWithPrefixSuffix( - dedupe(sampleResponse.data.flatMap(s => (s.parsed ? Object.keys(s.parsed) : []))), + dedupe(sampleResponse.data.flatMap(s => (s[key] ? Object.keys(s[key]!) : []))), columnOrder || [TIME_COLUMN], suffixColumnOrder || [], ); diff --git a/web-console/src/variables.scss b/web-console/src/variables.scss index d14c0b79a3d..86528e15661 100644 --- a/web-console/src/variables.scss +++ b/web-console/src/variables.scss @@ -35,7 +35,7 @@ $druid-brand-background: #1c1c26; background: $white; border-radius: $pt-border-radius; - .bp3-dark & { + .#{$bp-ns}-dark & { background: $dark-gray3; } } diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx index d04c630b536..b894f7d91c1 100644 --- a/web-console/src/views/datasource-view/datasource-view.tsx +++ b/web-console/src/views/datasource-view/datasource-view.tsx @@ -57,6 +57,7 @@ import { formatMillions, formatPercent, getDruidErrorMessage, + hasPopoverOpen, isNumberLikeNaN, LocalStorageBackedVisibility, LocalStorageKeys, @@ -493,7 +494,8 @@ ORDER BY 1`; }); } - private readonly refresh = (auto: any): void => { + private readonly refresh = (auto: boolean): void => { + if (auto && hasPopoverOpen()) return; this.datasourceQueryManager.rerunLastQuery(auto); this.tiersQueryManager.rerunLastQuery(auto); }; diff --git a/web-console/src/views/ingestion-view/ingestion-view.tsx b/web-console/src/views/ingestion-view/ingestion-view.tsx index 3071ebd4b65..ac4a8afe929 100644 --- a/web-console/src/views/ingestion-view/ingestion-view.tsx +++ b/web-console/src/views/ingestion-view/ingestion-view.tsx @@ -47,6 +47,7 @@ import { deepGet, formatDuration, getDruidErrorMessage, + hasPopoverOpen, LocalStorageBackedVisibility, localStorageGet, LocalStorageKeys, @@ -1089,7 +1090,10 @@ ORDER BY "rank" DESC, "created_time" DESC`; this.supervisorQueryManager.rerunLastQuery(auto)} + onRefresh={auto => { + if (auto && hasPopoverOpen()) return; + this.supervisorQueryManager.rerunLastQuery(auto); + }} /> {this.renderBulkSupervisorActions()} this.taskQueryManager.rerunLastQuery(auto)} + onRefresh={auto => { + if (auto && hasPopoverOpen()) return; + this.taskQueryManager.rerunLastQuery(auto); + }} /> {this.renderBulkTasksActions()} )} - {this.renderFlattenControls()} + {canFlatten && this.renderFlattenControls()} {suggestedFlattenFields && suggestedFlattenFields.length ? ( this.serviceQueryManager.rerunLastQuery(auto)} + onRefresh={auto => { + if (auto && hasPopoverOpen()) return; + this.serviceQueryManager.rerunLastQuery(auto); + }} localStorageKey={LocalStorageKeys.SERVICES_REFRESH_RATE} /> {this.renderBulkServicesActions()}