From ee61a165e3142149595362a01211227f2ce3b07a Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 30 Oct 2020 19:02:44 -0700 Subject: [PATCH] Web console: improve make type preservation in ingestion configs in the data loader (#10533) * improve validation * move to druid modals * adjust specs * oneOf * move transform * segment grans * tidy up webpack * add Partitioning * break out druid models * tidy up * rename to Expression * move druid time * cleanup * update format * better detail * fix e2e test * remove forceGuaranteedRollup from e2e tests * clean up imports * shardSpec -> shard_spec * fix css * adjust snapshot * add granularity to segments view * rename to Time span * use OFFSET in segments view query * update doc links * simplify require * more oneOf --- .../component/datasources/overview.ts | 7 +- .../component/load-data/config/partition.ts | 8 +- .../component/load-data/data-loader.ts | 14 +- .../e2e-tests/component/query/overview.ts | 3 +- web-console/e2e-tests/reindexing.spec.ts | 1 - web-console/e2e-tests/tutorial-batch.spec.ts | 1 - web-console/lib/keywords.js | 1 + .../src/components/auto-form/auto-form.scss | 4 - .../src/components/auto-form/auto-form.tsx | 46 +- .../__snapshots__/header-bar.spec.tsx.snap | 2 +- web-console/src/components/index.ts | 1 + .../src/components/json-input/json-input.tsx | 15 +- .../suggestible-input/suggestible-input.tsx | 1 + .../supervisor-statistics-table.tsx | 2 +- web-console/src/console-application.scss | 4 - web-console/src/console-application.tsx | 10 - .../compaction-dialog.spec.tsx.snap | 36 +- .../compaction-dialog/compaction-dialog.scss | 4 - .../compaction-dialog/compaction-dialog.tsx | 258 +---- ...inator-dynamic-config-dialog.spec.tsx.snap | 2 +- .../coordinator-dynamic-config-dialog.tsx | 2 +- .../dialogs/doctor-dialog/doctor-checks.tsx | 3 +- .../lookup-edit-dialog.spec.tsx.snap | 138 ++- .../lookup-edit-dialog.scss | 4 - .../lookup-edit-dialog.spec.tsx | 438 +------ .../lookup-edit-dialog/lookup-edit-dialog.tsx | 502 +------- ...erload-dynamic-config-dialog.spec.tsx.snap | 2 +- .../overlord-dynamic-config-dialog.tsx | 2 +- .../retention-dialog.spec.tsx.snap | 2 +- .../src/dialogs/spec-dialog/spec-dialog.scss | 4 - .../supervisor-table-action-dialog.tsx | 2 +- .../task-table-action-dialog.tsx | 2 +- .../__snapshots__/ingestion-spec.spec.ts.snap | 0 .../src/druid-models/compaction-config.tsx | 232 ++++ .../compaction-status.spec.ts} | 6 +- .../compaction-status.ts} | 4 +- .../src/druid-models/dimension-spec.spec.ts | 29 + .../src/druid-models/dimension-spec.ts | 84 ++ web-console/src/druid-models/filter.tsx | 121 ++ .../flatten-spec.spec.ts} | 2 +- .../flatten-spec.tsx} | 45 +- web-console/src/druid-models/index.ts | 31 + .../ingestion-spec.spec.ts | 106 +- .../ingestion-spec.tsx | 1021 +++-------------- web-console/src/druid-models/input-format.tsx | 131 +++ web-console/src/druid-models/input-source.tsx | 89 ++ .../src/druid-models/lookup-spec.spec.ts | 453 ++++++++ web-console/src/druid-models/lookup-spec.tsx | 456 ++++++++ .../src/druid-models/metric-spec.spec.ts | 32 + web-console/src/druid-models/metric-spec.tsx | 347 ++++++ .../time.spec.ts} | 2 +- .../druid-time.ts => druid-models/time.ts} | 2 +- .../src/druid-models/timestamp-spec.tsx | 157 +++ .../src/druid-models/transform-spec.tsx | 104 ++ web-console/src/entry.scss | 8 +- web-console/src/links.ts | 2 +- web-console/src/utils/druid-query.spec.ts | 16 +- web-console/src/utils/druid-type.ts | 115 -- web-console/src/utils/general.spec.ts | 28 +- web-console/src/utils/general.tsx | 18 +- web-console/src/utils/index.tsx | 3 +- web-console/src/utils/object-change.ts | 11 + web-console/src/utils/query-manager.tsx | 3 + web-console/src/utils/sampler.ts | 38 +- web-console/src/utils/utils.spec.ts | 170 +-- .../datasource-view.spec.tsx.snap | 14 + .../views/datasource-view/datasource-view.tsx | 74 +- .../home-view/segments-card/segments-card.tsx | 4 +- .../ingestion-view/ingestion-view.spec.tsx | 1 - .../views/ingestion-view/ingestion-view.tsx | 22 +- .../filter-table/filter-table.tsx | 2 +- .../views/load-data-view/load-data-view.scss | 13 +- .../views/load-data-view/load-data-view.tsx | 279 +++-- .../parse-data-table/parse-data-table.tsx | 2 +- .../parse-time-table.spec.tsx | 11 +- .../parse-time-table/parse-time-table.tsx | 49 +- .../schema-table/schema-table.tsx | 6 +- .../transform-table/transform-table.tsx | 2 +- .../src/views/lookups-view/lookups-view.tsx | 6 +- .../query-view/column-tree/column-tree.tsx | 6 +- .../query-view/query-output/query-output.tsx | 3 +- .../__snapshots__/segments-view.spec.tsx.snap | 18 + .../src/views/segments-view/segments-view.tsx | 190 +-- .../services-view/services-view.spec.tsx | 7 +- .../src/views/services-view/services-view.tsx | 25 +- web-console/webpack.config.js | 2 +- 86 files changed, 3325 insertions(+), 2798 deletions(-) rename web-console/src/{utils => druid-models}/__snapshots__/ingestion-spec.spec.ts.snap (100%) create mode 100644 web-console/src/druid-models/compaction-config.tsx rename web-console/src/{utils/compaction.spec.ts => druid-models/compaction-status.spec.ts} (95%) rename web-console/src/{utils/compaction.ts => druid-models/compaction-status.ts} (97%) create mode 100644 web-console/src/druid-models/dimension-spec.spec.ts create mode 100644 web-console/src/druid-models/dimension-spec.ts create mode 100644 web-console/src/druid-models/filter.tsx rename web-console/src/{utils/spec-utils.spec.ts => druid-models/flatten-spec.spec.ts} (97%) rename web-console/src/{utils/spec-utils.ts => druid-models/flatten-spec.tsx} (73%) create mode 100644 web-console/src/druid-models/index.ts rename web-console/src/{utils => druid-models}/ingestion-spec.spec.ts (60%) rename web-console/src/{utils => druid-models}/ingestion-spec.tsx (72%) create mode 100644 web-console/src/druid-models/input-format.tsx create mode 100644 web-console/src/druid-models/input-source.tsx create mode 100644 web-console/src/druid-models/lookup-spec.spec.ts create mode 100644 web-console/src/druid-models/lookup-spec.tsx create mode 100644 web-console/src/druid-models/metric-spec.spec.ts create mode 100644 web-console/src/druid-models/metric-spec.tsx rename web-console/src/{utils/druid-time.spec.ts => druid-models/time.spec.ts} (95%) rename web-console/src/{utils/druid-time.ts => druid-models/time.ts} (98%) create mode 100644 web-console/src/druid-models/timestamp-spec.tsx create mode 100644 web-console/src/druid-models/transform-spec.tsx delete mode 100644 web-console/src/utils/druid-type.ts diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index 5fb4bf4a3a7..f3f0c55a7f0 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -18,9 +18,7 @@ import * as playwright from 'playwright-chromium'; -import { clickButton } from '../../util/playwright'; -import { getLabeledInput } from '../../util/playwright'; -import { setLabeledInput } from '../../util/playwright'; +import { clickButton, getLabeledInput, setLabeledInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; import { readPartitionSpec } from '../load-data/config/partition'; @@ -36,10 +34,13 @@ enum DatasourceColumn { SEGMENT_LOAD_DROP, TOTAL_DATA_SIZE, SEGMENT_SIZE, + SEGMENT_GRANULARITY, TOTAL_ROWS, AVG_ROW_SIZE, REPLICATED_SIZE, COMPACTION, + PERCENT_COMPACTED, + LEFT_TO_BE_COMPACTED, RETENTION, ACTIONS, } diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts index 04e9d0e94c0..f811a531664 100644 --- a/web-console/e2e-tests/component/load-data/config/partition.ts +++ b/web-console/e2e-tests/component/load-data/config/partition.ts @@ -18,9 +18,7 @@ import * as playwright from 'playwright-chromium'; -import { selectSuggestibleInput } from '../../../util/playwright'; -import { getLabeledInput } from '../../../util/playwright'; -import { setLabeledInput } from '../../../util/playwright'; +import { getLabeledInput, selectSuggestibleInput, setLabeledInput } from '../../../util/playwright'; /* tslint:disable max-classes-per-file */ @@ -159,18 +157,14 @@ export interface SingleDimPartitionsSpec extends SingleDimPartitionsSpecProps {} * Data loader partition step configuration. */ export class PartitionConfig { - readonly forceGuaranteedRollupText: string; - constructor(props: PartitionConfigProps) { Object.assign(this, props); - this.forceGuaranteedRollupText = this.forceGuaranteedRollup ? 'True' : 'False'; } } interface PartitionConfigProps { readonly segmentGranularity: SegmentGranularity; readonly timeIntervals: string | null; - readonly forceGuaranteedRollup: boolean | null; readonly partitionsSpec: PartitionsSpec | null; } diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts index a0b64ff101b..df16e715875 100644 --- a/web-console/e2e-tests/component/load-data/data-loader.ts +++ b/web-console/e2e-tests/component/load-data/data-loader.ts @@ -18,10 +18,7 @@ import * as playwright from 'playwright-chromium'; -import { clickButton } from '../../util/playwright'; -import { clickLabeledButton } from '../../util/playwright'; -import { setLabeledInput } from '../../util/playwright'; -import { setLabeledTextarea } from '../../util/playwright'; +import { clickButton, setLabeledInput, setLabeledTextarea } from '../../util/playwright'; import { ConfigureSchemaConfig } from './config/configure-schema'; import { PartitionConfig } from './config/partition'; @@ -128,13 +125,8 @@ export class DataLoader { private async applyPartitionConfig(partitionConfig: PartitionConfig) { await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity); - if (partitionConfig.forceGuaranteedRollup) { - await clickLabeledButton( - this.page, - 'Force guaranteed rollup', - partitionConfig.forceGuaranteedRollupText, - ); - await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals!); + if (partitionConfig.timeIntervals) { + await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals); } if (partitionConfig.partitionsSpec != null) { await partitionConfig.partitionsSpec.apply(this.page); diff --git a/web-console/e2e-tests/component/query/overview.ts b/web-console/e2e-tests/component/query/overview.ts index 9e3c646cf37..d3b8986418d 100644 --- a/web-console/e2e-tests/component/query/overview.ts +++ b/web-console/e2e-tests/component/query/overview.ts @@ -18,8 +18,7 @@ import * as playwright from 'playwright-chromium'; -import { clickButton } from '../../util/playwright'; -import { setInput } from '../../util/playwright'; +import { clickButton, setInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; /** diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts index d3a02c1864f..ae45b735965 100644 --- a/web-console/e2e-tests/reindexing.spec.ts +++ b/web-console/e2e-tests/reindexing.spec.ts @@ -68,7 +68,6 @@ describe('Reindexing from Druid', () => { const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY, timeIntervals: interval, - forceGuaranteedRollup: true, partitionsSpec: new SingleDimPartitionsSpec({ partitionDimension: 'channel', targetRowsPerSegment: 10_000, diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 1fad5a9d23d..f4fa4505463 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -65,7 +65,6 @@ describe('Tutorial: Loading a file', () => { const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY, timeIntervals: null, - forceGuaranteedRollup: null, partitionsSpec: null, }); const publishConfig = new PublishConfig({ datasourceName: datasourceName }); diff --git a/web-console/lib/keywords.js b/web-console/lib/keywords.js index dfc2bd63c43..e476dfbd27a 100644 --- a/web-console/lib/keywords.js +++ b/web-console/lib/keywords.js @@ -36,6 +36,7 @@ exports.SQL_KEYWORDS = [ 'ASC', 'DESC', 'LIMIT', + 'OFFSET', 'UNION ALL', 'JOIN', 'LEFT', diff --git a/web-console/src/components/auto-form/auto-form.scss b/web-console/src/components/auto-form/auto-form.scss index 898f9b1495b..b880b55e94e 100644 --- a/web-console/src/components/auto-form/auto-form.scss +++ b/web-console/src/components/auto-form/auto-form.scss @@ -17,10 +17,6 @@ */ .auto-form { - .ace-solarized-dark { - background-color: #212e37; - } - // Popover in info label label.bp3-label { position: relative; diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index ce26cadf293..6ae9e4b3b4d 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -19,7 +19,7 @@ import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core'; import React from 'react'; -import { deepDelete, deepGet, deepSet } from '../../utils/object-change'; +import { deepDelete, deepGet, deepSet } from '../../utils'; import { ArrayInput } from '../array-input/array-input'; import { FormGroupWithInfo } from '../form-group-with-info/form-group-with-info'; import { IntervalInput } from '../interval-input/interval-input'; @@ -55,6 +55,7 @@ export interface Field { defined?: Functor; required?: Functor; adjustment?: (model: M) => M; + issueWithValue?: (value: any) => string | undefined; } export interface AutoFormProps { @@ -93,6 +94,48 @@ export class AutoForm> extends React.PureComponent } } + static issueWithModel(model: M | undefined, fields: readonly Field[]): string | undefined { + if (typeof model === 'undefined') { + return `model is undefined`; + } + + // Precompute which fields are defined because fields could be defined twice and only one should do the checking + const definedFields: Record> = {}; + for (const field of fields) { + const fieldDefined = AutoForm.evaluateFunctor(field.defined, model, true); + if (fieldDefined) { + definedFields[field.name] = field; + } + } + + for (const field of fields) { + const fieldValue = deepGet(model, field.name); + const fieldValueDefined = typeof fieldValue !== 'undefined'; + const fieldThatIsDefined = definedFields[field.name]; + if (fieldThatIsDefined) { + if (fieldThatIsDefined === field) { + const fieldRequired = AutoForm.evaluateFunctor(field.required, model, false); + if (fieldRequired) { + if (!fieldValueDefined) { + return `field ${field.name} is required`; + } + } + + if (fieldValueDefined && field.issueWithValue) { + const valueIssue = field.issueWithValue(fieldValue); + if (valueIssue) return `field ${field.name} has issue ${valueIssue}`; + } + } + } else { + // The field is undefined + if (fieldValueDefined) { + return `field ${field.name} is defined but it should not be`; + } + } + } + return; + } + constructor(props: AutoFormProps) { super(props); this.state = {}; @@ -274,6 +317,7 @@ export class AutoForm> extends React.PureComponent onChange={(v: any) => this.fieldChange(field, v)} placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')} height={field.height} + issueWithValue={field.issueWithValue} /> ); } diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index 788cabc41b7..0aadb182e0a 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -159,7 +159,7 @@ exports[`header bar matches snapshot 1`] = ` /> string | undefined; } export const JsonInput = React.memo(function JsonInput(props: JsonInputProps) { - const { onChange, placeholder, focus, width, height, value } = props; + const { onChange, placeholder, focus, width, height, value, issueWithValue } = props; const [internalValue, setInternalValue] = useState(() => ({ value, stringified: stringifyJson(value), @@ -102,6 +105,14 @@ export const JsonInput = React.memo(function JsonInput(props: JsonInputProps) { error = e; } + if (!error && issueWithValue) { + const issue = issueWithValue(value); + if (issue) { + value = undefined; + error = new Error(issue); + } + } + setInternalValue({ value, error, diff --git a/web-console/src/components/suggestible-input/suggestible-input.tsx b/web-console/src/components/suggestible-input/suggestible-input.tsx index c4e37fd1fb1..0b05917a228 100644 --- a/web-console/src/components/suggestible-input/suggestible-input.tsx +++ b/web-console/src/components/suggestible-input/suggestible-input.tsx @@ -83,6 +83,7 @@ export const SuggestibleInput = React.memo(function SuggestibleInput(props: Sugg rightElement={ suggestions && ( {suggestions.map(suggestion => { diff --git a/web-console/src/components/supervisor-statistics-table/supervisor-statistics-table.tsx b/web-console/src/components/supervisor-statistics-table/supervisor-statistics-table.tsx index a37059219aa..a8c5383bc08 100644 --- a/web-console/src/components/supervisor-statistics-table/supervisor-statistics-table.tsx +++ b/web-console/src/components/supervisor-statistics-table/supervisor-statistics-table.tsx @@ -23,7 +23,7 @@ import ReactTable, { CellInfo, Column } from 'react-table'; import { useQueryManager } from '../../hooks'; import { UrlBaser } from '../../singletons/url-baser'; -import { deepGet } from '../../utils/object-change'; +import { deepGet } from '../../utils'; import { Loader } from '../loader/loader'; import './supervisor-statistics-table.scss'; diff --git a/web-console/src/console-application.scss b/web-console/src/console-application.scss index d3015b7cc5e..7f6107e86b9 100644 --- a/web-console/src/console-application.scss +++ b/web-console/src/console-application.scss @@ -46,8 +46,4 @@ height: 22px; border-top: 2px solid #6d8ea9; } - - .ace-solarized-dark { - background-color: rgba($dark-gray1, 0.5); - } } diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx index c8b9001d47f..c09100a6a77 100644 --- a/web-console/src/console-application.tsx +++ b/web-console/src/console-application.tsx @@ -77,7 +77,6 @@ export class ConsoleApplication extends React.PureComponent< private datasource?: string; private onlyUnavailable?: boolean; private initQuery?: string; - private middleManager?: string; constructor(props: ConsoleApplicationProps, context: any) { super(props, context); @@ -118,7 +117,6 @@ export class ConsoleApplication extends React.PureComponent< this.datasource = undefined; this.onlyUnavailable = undefined; this.initQuery = undefined; - this.middleManager = undefined; }, 50); } @@ -156,12 +154,6 @@ export class ConsoleApplication extends React.PureComponent< this.resetInitialsWithDelay(); }; - private goToMiddleManager = (middleManager: string) => { - this.middleManager = middleManager; - window.location.hash = 'services'; - this.resetInitialsWithDelay(); - }; - private goToQuery = (initQuery: string) => { this.initQuery = initQuery; window.location.hash = 'query'; @@ -254,7 +246,6 @@ export class ConsoleApplication extends React.PureComponent< openDialog={this.openDialog} goToDatasource={this.goToDatasources} goToQuery={this.goToQuery} - goToMiddleManager={this.goToMiddleManager} goToLoadData={this.goToLoadData} capabilities={capabilities} />, @@ -266,7 +257,6 @@ export class ConsoleApplication extends React.PureComponent< return this.wrapInViewContainer( 'services', , "label": "Partition dimensions", "name": "tuningConfig.partitionsSpec.partitionDimensions", + "placeholder": "(all dimensions)", "type": "string-array", }, Object { @@ -175,14 +178,14 @@ exports[`CompactionDialog matches snapshot with compactionConfig (dynamic partit "type": "number", }, Object { - "defaultValue": 1, + "defaultValue": 10, "defined": [Function], "info": Maximum number of merge tasks which can be run at the same time. , - "label": "Max num merge tasks", + "label": "Total num merge tasks", "min": 1, - "name": "tuningConfig.maxNumMergeTasks", + "name": "tuningConfig.totalNumMergeTasks", "type": "number", }, Object { @@ -327,6 +330,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (hashed partiti "label": "Target rows per segment", "name": "tuningConfig.partitionsSpec.targetRowsPerSegment", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -341,6 +345,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (hashed partiti "label": "Num shards", "name": "tuningConfig.partitionsSpec.numShards", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -349,6 +354,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (hashed partiti

, "label": "Partition dimensions", "name": "tuningConfig.partitionsSpec.partitionDimensions", + "placeholder": "(all dimensions)", "type": "string-array", }, Object { @@ -412,14 +418,14 @@ exports[`CompactionDialog matches snapshot with compactionConfig (hashed partiti "type": "number", }, Object { - "defaultValue": 1, + "defaultValue": 10, "defined": [Function], "info": Maximum number of merge tasks which can be run at the same time. , - "label": "Max num merge tasks", + "label": "Total num merge tasks", "min": 1, - "name": "tuningConfig.maxNumMergeTasks", + "name": "tuningConfig.totalNumMergeTasks", "type": "number", }, Object { @@ -564,6 +570,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (single_dim par "label": "Target rows per segment", "name": "tuningConfig.partitionsSpec.targetRowsPerSegment", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -578,6 +585,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (single_dim par "label": "Num shards", "name": "tuningConfig.partitionsSpec.numShards", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -586,6 +594,7 @@ exports[`CompactionDialog matches snapshot with compactionConfig (single_dim par

, "label": "Partition dimensions", "name": "tuningConfig.partitionsSpec.partitionDimensions", + "placeholder": "(all dimensions)", "type": "string-array", }, Object { @@ -649,14 +658,14 @@ exports[`CompactionDialog matches snapshot with compactionConfig (single_dim par "type": "number", }, Object { - "defaultValue": 1, + "defaultValue": 10, "defined": [Function], "info": Maximum number of merge tasks which can be run at the same time. , - "label": "Max num merge tasks", + "label": "Total num merge tasks", "min": 1, - "name": "tuningConfig.maxNumMergeTasks", + "name": "tuningConfig.totalNumMergeTasks", "type": "number", }, Object { @@ -801,6 +810,7 @@ exports[`CompactionDialog matches snapshot without compactionConfig 1`] = ` "label": "Target rows per segment", "name": "tuningConfig.partitionsSpec.targetRowsPerSegment", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -815,6 +825,7 @@ exports[`CompactionDialog matches snapshot without compactionConfig 1`] = ` "label": "Num shards", "name": "tuningConfig.partitionsSpec.numShards", "type": "number", + "zeroMeansUndefined": true, }, Object { "defined": [Function], @@ -823,6 +834,7 @@ exports[`CompactionDialog matches snapshot without compactionConfig 1`] = `

, "label": "Partition dimensions", "name": "tuningConfig.partitionsSpec.partitionDimensions", + "placeholder": "(all dimensions)", "type": "string-array", }, Object { @@ -886,14 +898,14 @@ exports[`CompactionDialog matches snapshot without compactionConfig 1`] = ` "type": "number", }, Object { - "defaultValue": 1, + "defaultValue": 10, "defined": [Function], "info": Maximum number of merge tasks which can be run at the same time. , - "label": "Max num merge tasks", + "label": "Total num merge tasks", "min": 1, - "name": "tuningConfig.maxNumMergeTasks", + "name": "tuningConfig.totalNumMergeTasks", "type": "number", }, Object { diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss index f5cd57ff384..00cea76f381 100644 --- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss +++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss @@ -31,8 +31,4 @@ flex: 1; overflow: auto; } - - .ace-solarized-dark { - background-color: #232c35; - } } diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx index d2b805526bc..8c96e9765f9 100644 --- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx +++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx @@ -16,254 +16,14 @@ * limitations under the License. */ -import { Button, Classes, Code, Dialog, Intent } from '@blueprintjs/core'; +import { Button, Classes, Dialog, Intent } from '@blueprintjs/core'; import React, { useState } from 'react'; -import { AutoForm, Field, JsonInput } from '../../components'; -import { - FormJsonSelector, - FormJsonTabs, -} from '../../components/form-json-selector/form-json-selector'; -import { deepGet, deepSet } from '../../utils/object-change'; +import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components'; +import { COMPACTION_CONFIG_FIELDS, CompactionConfig } from '../../druid-models'; import './compaction-dialog.scss'; -type CompactionConfig = Record; - -const COMPACTION_CONFIG_FIELDS: Field[] = [ - { - name: 'skipOffsetFromLatest', - type: 'string', - defaultValue: 'P1D', - suggestions: ['PT0H', 'PT1H', 'P1D', 'P3D'], - info: ( -

- The offset for searching segments to be compacted. Strongly recommended to set for realtime - dataSources. -

- ), - }, - { - name: 'tuningConfig.partitionsSpec.type', - label: 'Partitioning type', - type: 'string', - suggestions: ['dynamic', 'hashed', 'single_dim'], - info: ( -

- For perfect rollup, you should use either hashed (partitioning based on the - hash of dimensions in each row) or single_dim (based on ranges of a single - dimension). For best-effort rollup, you should use dynamic. -

- ), - }, - // partitionsSpec type: dynamic - { - name: 'tuningConfig.partitionsSpec.maxRowsPerSegment', - label: 'Max rows per segment', - type: 'number', - defaultValue: 5000000, - defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic', - info: <>Determines how many rows are in each segment., - }, - { - name: 'tuningConfig.partitionsSpec.maxTotalRows', - label: 'Max total rows', - type: 'number', - defaultValue: 20000000, - defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic', - info: <>Total number of rows in segments waiting for being pushed., - }, - // partitionsSpec type: hashed - { - name: 'tuningConfig.partitionsSpec.targetRowsPerSegment', - label: 'Target rows per segment', - type: 'number', - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' && - !deepGet(t, 'tuningConfig.partitionsSpec.numShards'), - info: ( - <> -

- If the segments generated are a sub-optimal size for the requested partition dimensions, - consider setting this field. -

-

- A target row count for each partition. Each partition will have a row count close to the - target assuming evenly distributed keys. Defaults to 5 million if numShards is null. -

- - ), - }, - { - name: 'tuningConfig.partitionsSpec.numShards', - label: 'Num shards', - type: 'number', - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' && - !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'), - info: ( - <> -

- If you know the optimal number of shards and want to speed up the time it takes for - compaction to run, set this field. -

-

- Directly specify the number of shards to create. If this is specified and 'intervals' is - specified in the granularitySpec, the index task can skip the determine - intervals/partitions pass through the data. -

- - ), - }, - { - name: 'tuningConfig.partitionsSpec.partitionDimensions', - label: 'Partition dimensions', - type: 'string-array', - defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed', - info:

The dimensions to partition on. Leave blank to select all dimensions.

, - }, - // partitionsSpec type: single_dim - { - name: 'tuningConfig.partitionsSpec.partitionDimension', - label: 'Partition dimension', - type: 'string', - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim', - required: true, - info:

The dimension to partition on.

, - }, - { - name: 'tuningConfig.partitionsSpec.targetRowsPerSegment', - label: 'Target rows per segment', - type: 'number', - zeroMeansUndefined: true, - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' && - !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), - required: (t: CompactionConfig) => - !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') && - !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), - info: ( -

- Target number of rows to include in a partition, should be a number that targets segments of - 500MB~1GB. -

- ), - }, - { - name: 'tuningConfig.partitionsSpec.maxRowsPerSegment', - label: 'Max rows per segment', - type: 'number', - zeroMeansUndefined: true, - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' && - !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'), - required: (t: CompactionConfig) => - !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') && - !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), - info:

Maximum number of rows to include in a partition.

, - }, - { - name: 'tuningConfig.partitionsSpec.assumeGrouped', - label: 'Assume grouped', - type: 'boolean', - defaultValue: false, - defined: (t: CompactionConfig) => - deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim', - info: ( -

- Assume that input data has already been grouped on time and dimensions. Ingestion will run - faster, but may choose sub-optimal partitions if this assumption is violated. -

- ), - }, - { - name: 'tuningConfig.maxNumConcurrentSubTasks', - label: 'Max num concurrent sub tasks', - type: 'number', - defaultValue: 1, - min: 1, - info: ( - <> - Maximum number of tasks which can be run at the same time. The supervisor task would spawn - worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this - value is set to 1, the supervisor task processes data ingestion on its own instead of - spawning worker tasks. If this value is set to too large, too many worker tasks can be - created which might block other ingestion. - - ), - }, - { - name: 'inputSegmentSizeBytes', - type: 'number', - defaultValue: 419430400, - info: ( -

- Maximum number of total segment bytes processed per compaction task. Since a time chunk must - be processed in its entirety, if the segments for a particular time chunk have a total size - in bytes greater than this parameter, compaction will not run for that time chunk. Because - each compaction task runs with a single thread, setting this value too far above 1–2GB will - result in compaction tasks taking an excessive amount of time. -

- ), - }, - { - name: 'tuningConfig.maxNumMergeTasks', - label: 'Max num merge tasks', - type: 'number', - defaultValue: 1, - min: 1, - defined: (t: CompactionConfig) => - ['hashed', 'single_dim'].includes(deepGet(t, 'tuningConfig.partitionsSpec.type')), - info: <>Maximum number of merge tasks which can be run at the same time., - }, - { - name: 'tuningConfig.splitHintSpec.maxInputSegmentBytesPerTask', - label: 'Max input segment bytes per task', - type: 'number', - defaultValue: 500000000, - min: 1000000, - adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'segments'), - info: ( - <> - Maximum number of bytes of input segments to process in a single task. If a single segment - is larger than this number, it will be processed by itself in a single task (input segments - are never split across tasks). - - ), - }, -]; - -function validCompactionConfig(compactionConfig: CompactionConfig): boolean { - const partitionsSpecType = - deepGet(compactionConfig, 'tuningConfig.partitionsSpec.type') || 'dynamic'; - switch (partitionsSpecType) { - // case 'dynamic': // Nothing to check for dynamic - case 'hashed': - return !( - Boolean(deepGet(compactionConfig, 'tuningConfig.partitionsSpec.targetRowsPerSegment')) && - Boolean(deepGet(compactionConfig, 'tuningConfig.partitionsSpec.numShards')) - ); - break; - case 'single_dim': - if (!deepGet(compactionConfig, 'tuningConfig.partitionsSpec.partitionDimension')) { - return false; - } - const hasTargetRowsPerSegment = Boolean( - deepGet(compactionConfig, 'tuningConfig.partitionsSpec.targetRowsPerSegment'), - ); - const hasMaxRowsPerSegment = Boolean( - deepGet(compactionConfig, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), - ); - if (hasTargetRowsPerSegment === hasMaxRowsPerSegment) { - return false; - } - break; - } - - return true; -} - export interface CompactionDialogProps { onClose: () => void; onSave: (compactionConfig: CompactionConfig) => void; @@ -283,8 +43,9 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp }, ); + const issueWithCurrentConfig = AutoForm.issueWithModel(currentConfig, COMPACTION_CONFIG_FIELDS); function handleSubmit() { - if (!validCompactionConfig(currentConfig)) return; + if (issueWithCurrentConfig) return; onSave(currentConfig); } @@ -305,7 +66,12 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp onChange={m => setCurrentConfig(m)} /> ) : ( - + AutoForm.issueWithModel(value, COMPACTION_CONFIG_FIELDS)} + height="100%" + /> )}
@@ -316,7 +82,7 @@ export const CompactionDialog = React.memo(function CompactionDialog(props: Comp text="Submit" intent={Intent.PRIMARY} onClick={handleSubmit} - disabled={!validCompactionConfig(currentConfig)} + disabled={Boolean(issueWithCurrentConfig)} />
diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap index dc3cdacd610..4363077e56c 100644 --- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap @@ -11,7 +11,7 @@ exports[`coordinator dynamic config matches snapshot 1`] = ` Edit the coordinator dynamic configuration on the fly. For more information please refer to the documentation diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx index 8478ec6a39a..f3f71b8d79b 100644 --- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx @@ -52,7 +52,7 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn processQuery: async () => { try { const configResp = await axios.get('/druid/coordinator/v1/config'); - setDynamicConfig(configResp.data); + setDynamicConfig(configResp.data || {}); } catch (e) { AppToaster.show({ icon: IconNames.ERROR, diff --git a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx index 1dd546886c9..a5ecb853616 100644 --- a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx +++ b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx @@ -18,8 +18,7 @@ import axios from 'axios'; -import { pluralIfNeeded, queryDruidSql } from '../../utils'; -import { deepGet } from '../../utils/object-change'; +import { deepGet, pluralIfNeeded, queryDruidSql } from '../../utils'; import { postToSampler } from '../../utils/sampler'; export interface CheckControls { diff --git a/web-console/src/dialogs/lookup-edit-dialog/__snapshots__/lookup-edit-dialog.spec.tsx.snap b/web-console/src/dialogs/lookup-edit-dialog/__snapshots__/lookup-edit-dialog.spec.tsx.snap index 5349950703f..7b7dabc71a8 100644 --- a/web-console/src/dialogs/lookup-edit-dialog/__snapshots__/lookup-edit-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/lookup-edit-dialog/__snapshots__/lookup-edit-dialog.spec.tsx.snap @@ -16,6 +16,7 @@ exports[`LookupEditDialog matches snapshot 1`] = ` > +

+ URI for the file of interest, specified as a file, hdfs, or s3 path +

+

+ The URI prefix option is strictly better than URI and should be used instead +

+ , + "label": "URI (deprecated)", + "name": "extractionNamespace.uri", + "placeholder": "s3://bucket/some/key/prefix/lookups-01.gz", + "required": [Function], + "type": "string", + }, + Object { + "defaultValue": ".*", + "defined": [Function], + "info": "Optional regex for matching the file name under uriPrefix.", "label": "File regex", "name": "extractionNamespace.fileRegex", - "placeholder": "(optional)", "type": "string", }, Object { - "defaultValue": "csv", "defined": [Function], - "label": "Format", + "info": +

+ The format of the data in the lookup files. +

+

+ The + + simpleJson + + lookupParseSpec does not take any parameters. It is simply a line delimited JSON file where the field is the key, and the field's value is the value. +

+ , + "label": "Parse format", "name": "extractionNamespace.namespaceParseSpec.format", + "required": true, "suggestions": Array [ "csv", "tsv", - "customJson", "simpleJson", + "customJson", ], "type": "string", }, Object { + "defaultValue": 0, "defined": [Function], - "info": "The list of columns in the csv file", - "label": "Columns", - "name": "extractionNamespace.namespaceParseSpec.columns", - "placeholder": "[\\"key\\", \\"value\\"]", - "type": "string-array", - }, - Object { - "defined": [Function], - "info": "The name of the column containing the key", - "label": "Key column", - "name": "extractionNamespace.namespaceParseSpec.keyColumn", - "placeholder": "Key", - "type": "string", - }, - Object { - "defined": [Function], - "info": "The name of the column containing the value", - "label": "Value column", - "name": "extractionNamespace.namespaceParseSpec.valueColumn", - "placeholder": "Value", - "type": "string", + "info": "Number of header rows to be skipped. The default number of header rows to be skipped is 0.", + "label": "Skip header rows", + "name": "extractionNamespace.namespaceParseSpec.skipHeaderRows", + "type": "number", }, Object { "defaultValue": false, @@ -154,11 +172,28 @@ exports[`LookupEditDialog matches snapshot 1`] = ` }, Object { "defined": [Function], - "info": "Number of header rows to be skipped. The default number of header rows to be skipped is 0.", - "label": "Skip header rows", - "name": "extractionNamespace.namespaceParseSpec.skipHeaderRows", - "placeholder": "(optional)", - "type": "number", + "info": "The list of columns in the csv file", + "label": "Columns", + "name": "extractionNamespace.namespaceParseSpec.columns", + "placeholder": "[\\"key\\", \\"value\\"]", + "required": [Function], + "type": "string-array", + }, + Object { + "defined": [Function], + "info": "The name of the column containing the key", + "label": "Key column", + "name": "extractionNamespace.namespaceParseSpec.keyColumn", + "placeholder": "(optional - defaults to the first column)", + "type": "string", + }, + Object { + "defined": [Function], + "info": "The name of the column containing the value", + "label": "Value column", + "name": "extractionNamespace.namespaceParseSpec.valueColumn", + "placeholder": "(optional - defaults to the second column)", + "type": "string", }, Object { "defined": [Function], @@ -179,6 +214,7 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Key field name", "name": "extractionNamespace.namespaceParseSpec.keyFieldName", "placeholder": "key", + "required": true, "type": "string", }, Object { @@ -186,6 +222,15 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Value field name", "name": "extractionNamespace.namespaceParseSpec.valueFieldName", "placeholder": "value", + "required": true, + "type": "string", + }, + Object { + "defaultValue": "0", + "defined": [Function], + "info": "Period between polling for updates", + "label": "Poll period", + "name": "extractionNamespace.pollPeriod", "type": "string", }, Object { @@ -205,20 +250,15 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Namespace", "name": "extractionNamespace.namespace", "placeholder": "some_lookup", + "required": true, "type": "string", }, - Object { - "defined": [Function], - "info": "Defines the connectURI value on the The connector config to used", - "label": "CreateTables", - "name": "extractionNamespace.connectorConfig.createTables", - "type": "boolean", - }, Object { "defined": [Function], "info": "Defines the connectURI value on the The connector config to used", "label": "Connect URI", "name": "extractionNamespace.connectorConfig.connectURI", + "required": true, "type": "string", }, Object { @@ -235,6 +275,13 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "name": "extractionNamespace.connectorConfig.password", "type": "string", }, + Object { + "defined": [Function], + "info": "Should tables be created", + "label": "Create tables", + "name": "extractionNamespace.connectorConfig.createTables", + "type": "boolean", + }, Object { "defined": [Function], "info": @@ -252,6 +299,7 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Table", "name": "extractionNamespace.table", "placeholder": "some_lookup_table", + "required": true, "type": "string", }, Object { @@ -271,6 +319,7 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Key column", "name": "extractionNamespace.keyColumn", "placeholder": "my_key_value", + "required": true, "type": "string", }, Object { @@ -290,6 +339,7 @@ exports[`LookupEditDialog matches snapshot 1`] = ` "label": "Value column", "name": "extractionNamespace.valueColumn", "placeholder": "my_column_value", + "required": true, "type": "string", }, Object { @@ -325,25 +375,17 @@ exports[`LookupEditDialog matches snapshot 1`] = ` ? FROM namespace.table WHERE filter

, - "label": "TsColumn", + "label": "Timestamp column", "name": "extractionNamespace.tsColumn", "placeholder": "(optional)", "type": "string", }, Object { - "defined": [Function], - "info": "Period between polling for updates", - "label": "Poll period", - "name": "extractionNamespace.pollPeriod", - "placeholder": "(optional)", - "type": "string", - }, - Object { + "defaultValue": 0, "defined": [Function], "info": "How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait", "label": "First cache timeout", "name": "firstCacheTimeout", - "placeholder": "(optional)", "type": "number", }, Object { diff --git a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.scss b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.scss index e42914f54cb..ce69aae004a 100644 --- a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.scss +++ b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.scss @@ -29,10 +29,6 @@ overflow: auto; } - .ace-solarized-dark { - background-color: #232c35; - } - .ace_gutter-layer { background-color: #27313c; } diff --git a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.spec.tsx b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.spec.tsx index f9eebd800c5..432d53dc407 100644 --- a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.spec.tsx +++ b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.spec.tsx @@ -19,7 +19,7 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { isLookupSubmitDisabled, LookupEditDialog } from './lookup-edit-dialog'; +import { LookupEditDialog } from './lookup-edit-dialog'; describe('LookupEditDialog', () => { it('matches snapshot', () => { @@ -40,439 +40,3 @@ describe('LookupEditDialog', () => { expect(lookupEditDialog).toMatchSnapshot(); }); }); - -describe('Type Map Should be disabled', () => { - it('Missing LookupName', () => { - expect(isLookupSubmitDisabled(undefined, 'v1', '__default', { type: '' })).toBe(true); - }); - - it('Empty version', () => { - expect(isLookupSubmitDisabled('lookup', '', '__default', { type: '' })).toBe(true); - }); - - it('Missing version', () => { - expect(isLookupSubmitDisabled('lookup', undefined, '__default', { type: '' })).toBe(true); - }); - - it('Empty tier', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '', { type: '' })).toBe(true); - }); - - it('Missing tier', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', undefined, { type: '' })).toBe(true); - }); - - it('Missing spec', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '__default', {})).toBe(true); - }); - - it('Type undefined', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '__default', { type: undefined })).toBe(true); - }); - - it('Lookup of type map with no map', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '__default', { type: 'map' })).toBe(true); - }); - - it('Lookup of type cachedNamespace with no extractionNamespace', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '__default', { type: 'cachedNamespace' })).toBe( - true, - ); - }); - - it('Lookup of type cachedNamespace with extractionNamespace type uri, format csv, no namespaceParseSpec', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Lookup of type cachedNamespace with extractionNamespace type uri, format csv, no columns and skipHeaderRows', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'csv', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Lookup of type cachedNamespace with extractionNamespace type uri, format tsv, no columns', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'tsv', - skipHeaderRows: 0, - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Lookup of type cachedNamespace with extractionNamespace type customJson, format tsv, no keyFieldName', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'customJson', - valueFieldName: 'value', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Lookup of type cachedNamespace with extractionNamespace type customJson, format customJson, no valueFieldName', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'customJson', - keyFieldName: 'key', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); -}); - -describe('Type cachedNamespace should be disabled', () => { - it('No extractionNamespace', () => { - expect(isLookupSubmitDisabled('lookup', 'v1', '__default', { type: 'cachedNamespace' })).toBe( - true, - ); - }); - - describe('ExtractionNamespace type URI', () => { - it('Format csv, no namespaceParseSpec', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Format csv, no columns and skipHeaderRows', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'csv', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Format tsv, no columns', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'tsv', - skipHeaderRows: 0, - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Format tsv, no keyFieldName', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'customJson', - valueFieldName: 'value', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - - it('Format customJson, no valueFieldName', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'customJson', - keyFieldName: 'key', - }, - pollPeriod: 'PT5M', - }, - }), - ).toBe(true); - }); - }); - - describe('ExtractionNamespace type JDBC', () => { - it('No namespace', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: undefined, - connectorConfig: { - createTables: true, - connectURI: 'jdbc:mysql://localhost:3306/druid', - user: 'druid', - password: 'diurd', - }, - table: 'some_lookup_table', - keyColumn: 'the_old_dim_value', - valueColumn: 'the_new_dim_value', - tsColumn: 'timestamp_column', - pollPeriod: 600000, - }, - }), - ).toBe(true); - }); - - it('No connectorConfig', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: 'some_lookup', - connectorConfig: undefined, - table: 'some_lookup_table', - keyColumn: 'the_old_dim_value', - valueColumn: 'the_new_dim_value', - tsColumn: 'timestamp_column', - pollPeriod: 600000, - }, - }), - ).toBe(true); - }); - - it('No table', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: 'some_lookup', - connectorConfig: { - createTables: true, - connectURI: 'jdbc:mysql://localhost:3306/druid', - user: 'druid', - password: 'diurd', - }, - table: undefined, - keyColumn: 'the_old_dim_value', - valueColumn: 'the_new_dim_value', - tsColumn: 'timestamp_column', - pollPeriod: 600000, - }, - }), - ).toBe(true); - }); - - it('No keyColumn', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: 'some_lookup', - connectorConfig: { - createTables: true, - connectURI: 'jdbc:mysql://localhost:3306/druid', - user: 'druid', - password: 'diurd', - }, - table: 'some_lookup_table', - keyColumn: undefined, - valueColumn: 'the_new_dim_value', - tsColumn: 'timestamp_column', - pollPeriod: 600000, - }, - }), - ).toBe(true); - }); - - it('No keyColumn', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: 'some_lookup', - connectorConfig: { - createTables: true, - connectURI: 'jdbc:mysql://localhost:3306/druid', - user: 'druid', - password: 'diurd', - }, - table: 'some_lookup_table', - keyColumn: 'the_old_dim_value', - valueColumn: undefined, - tsColumn: 'timestamp_column', - pollPeriod: 600000, - }, - }), - ).toBe(true); - }); - }); -}); - -describe('Type Map Should be enabled', () => { - it('Has type and has Map', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { type: 'map', map: { a: 'b' } }), - ).toBe(false); - }); -}); - -describe('Type cachedNamespace Should be enabled', () => { - describe('ExtractionNamespace type URI', () => { - it('Format csv with columns', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'csv', - columns: ['key', 'value'], - }, - }, - }), - ).toBe(false); - }); - - it('Format csv with skipHeaderRows', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'csv', - skipHeaderRows: 1, - }, - }, - }), - ).toBe(false); - }); - - it('Format tsv, only columns', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'tsv', - columns: ['key', 'value'], - }, - }, - }), - ).toBe(false); - }); - - it('Format tsv, keyFieldName and valueFieldName', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'uri', - uriPrefix: 's3://bucket/some/key/prefix/', - fileRegex: 'renames-[0-9]*\\.gz', - namespaceParseSpec: { - format: 'customJson', - valueFieldName: 'value', - keyFieldName: 'value', - }, - }, - }), - ).toBe(false); - }); - }); - - describe('ExtractionNamespace type JDBC', () => { - it('No namespace', () => { - expect( - isLookupSubmitDisabled('lookup', 'v1', '__default', { - type: 'cachedNamespace', - extractionNamespace: { - type: 'jdbc', - namespace: 'lookup', - connectorConfig: { - createTables: true, - connectURI: 'jdbc:mysql://localhost:3306/druid', - user: 'druid', - password: 'diurd', - }, - table: 'some_lookup_table', - keyColumn: 'the_old_dim_value', - valueColumn: 'the_new_dim_value', - }, - }), - ).toBe(false); - }); - }); -}); diff --git a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx index 6c7061ef63c..11b2be83b1d 100644 --- a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx +++ b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx @@ -27,56 +27,12 @@ import { } from '@blueprintjs/core'; import React, { useState } from 'react'; -import { AutoForm, Field, JsonInput } from '../../components'; -import { - FormJsonSelector, - FormJsonTabs, -} from '../../components/form-json-selector/form-json-selector'; +import { AutoForm, JsonInput } from '../../components'; +import { FormJsonSelector, FormJsonTabs } from '../../components'; +import { isLookupInvalid, LOOKUP_FIELDS, LookupSpec } from '../../druid-models'; import './lookup-edit-dialog.scss'; -export interface ExtractionNamespaceSpec { - type?: string; - uri?: string; - uriPrefix?: string; - fileRegex?: string; - namespaceParseSpec?: NamespaceParseSpec; - namespace?: string; - connectorConfig?: { - createTables: boolean; - connectURI: string; - user: string; - password: string; - }; - table?: string; - keyColumn?: string; - valueColumn?: string; - filter?: any; - tsColumn?: string; - pollPeriod?: number | string; -} - -export interface NamespaceParseSpec { - format: string; - columns?: string[]; - keyColumn?: string; - valueColumn?: string; - hasHeaderRow?: boolean; - skipHeaderRows?: number; - keyFieldName?: string; - valueFieldName?: string; - delimiter?: string; - listDelimiter?: string; -} - -export interface LookupSpec { - type?: string; - map?: {}; - extractionNamespace?: ExtractionNamespaceSpec; - firstCacheTimeout?: number; - injective?: boolean; -} - export interface LookupEditDialogProps { onClose: () => void; onSubmit: (updateLookupVersion: boolean) => void; @@ -89,455 +45,6 @@ export interface LookupEditDialogProps { allLookupTiers: string[]; } -export function isLookupSubmitDisabled( - lookupName: string | undefined, - lookupVersion: string | undefined, - lookupTier: string | undefined, - lookupSpec: LookupSpec | undefined, -) { - let disableSubmit = - !lookupName || - !lookupVersion || - !lookupTier || - !lookupSpec || - !lookupSpec.type || - (lookupSpec.type === 'map' && !lookupSpec.map) || - (lookupSpec.type === 'cachedNamespace' && !lookupSpec.extractionNamespace); - - if ( - !disableSubmit && - lookupSpec && - lookupSpec.type === 'cachedNamespace' && - lookupSpec.extractionNamespace - ) { - switch (lookupSpec.extractionNamespace.type) { - case 'uri': - const namespaceParseSpec = lookupSpec.extractionNamespace.namespaceParseSpec; - disableSubmit = !namespaceParseSpec; - if (!namespaceParseSpec) break; - switch (namespaceParseSpec.format) { - case 'csv': - disableSubmit = !namespaceParseSpec.columns && !namespaceParseSpec.skipHeaderRows; - break; - case 'tsv': - disableSubmit = !namespaceParseSpec.columns; - break; - case 'customJson': - disableSubmit = !namespaceParseSpec.keyFieldName || !namespaceParseSpec.valueFieldName; - break; - } - break; - case 'jdbc': - const extractionNamespace = lookupSpec.extractionNamespace; - disableSubmit = - !extractionNamespace.namespace || - !extractionNamespace.connectorConfig || - !extractionNamespace.table || - !extractionNamespace.keyColumn || - !extractionNamespace.valueColumn; - break; - } - } - return disableSubmit; -} - -const LOOKUP_FIELDS: Field[] = [ - { - name: 'type', - type: 'string', - suggestions: ['map', 'cachedNamespace'], - adjustment: (model: LookupSpec) => { - if (model.type === 'map' && model.extractionNamespace && model.extractionNamespace.type) { - return model; - } - model.extractionNamespace = { type: 'uri', namespaceParseSpec: { format: 'csv' } }; - return model; - }, - }, - { - name: 'map', - type: 'json', - height: '60vh', - defined: (model: LookupSpec) => model.type === 'map', - }, - { - name: 'extractionNamespace.type', - type: 'string', - label: 'Globally cached lookup type', - placeholder: 'uri', - suggestions: ['uri', 'jdbc'], - defined: (model: LookupSpec) => model.type === 'cachedNamespace', - }, - { - name: 'extractionNamespace.uriPrefix', - type: 'string', - label: 'URI prefix', - info: - 'A URI which specifies a directory (or other searchable resource) in which to search for files', - placeholder: 's3://bucket/some/key/prefix/', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri', - }, - { - name: 'extractionNamespace.fileRegex', - type: 'string', - label: 'File regex', - placeholder: '(optional)', - info: - 'Optional regex for matching the file name under uriPrefix. Only used if uriPrefix is used', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri', - }, - { - name: 'extractionNamespace.namespaceParseSpec.format', - type: 'string', - label: 'Format', - defaultValue: 'csv', - suggestions: ['csv', 'tsv', 'customJson', 'simpleJson'], - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - model.extractionNamespace && - model.extractionNamespace.type === 'uri', - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.columns', - type: 'string-array', - label: 'Columns', - placeholder: `["key", "value"]`, - info: 'The list of columns in the csv file', - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - (model.extractionNamespace.namespaceParseSpec.format === 'csv' || - model.extractionNamespace.namespaceParseSpec.format === 'tsv'), - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.keyColumn', - type: 'string', - label: 'Key column', - placeholder: 'Key', - info: 'The name of the column containing the key', - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - (model.extractionNamespace.namespaceParseSpec.format === 'csv' || - model.extractionNamespace.namespaceParseSpec.format === 'tsv'), - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.valueColumn', - type: 'string', - label: 'Value column', - placeholder: 'Value', - info: 'The name of the column containing the value', - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - (model.extractionNamespace.namespaceParseSpec.format === 'csv' || - model.extractionNamespace.namespaceParseSpec.format === 'tsv'), - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.hasHeaderRow', - type: 'boolean', - label: 'Has header row', - defaultValue: false, - info: `A flag to indicate that column information can be extracted from the input files' header row`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - (model.extractionNamespace.namespaceParseSpec.format === 'csv' || - model.extractionNamespace.namespaceParseSpec.format === 'tsv'), - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.skipHeaderRows', - type: 'number', - label: 'Skip header rows', - placeholder: '(optional)', - info: `Number of header rows to be skipped. The default number of header rows to be skipped is 0.`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - (model.extractionNamespace.namespaceParseSpec.format === 'csv' || - model.extractionNamespace.namespaceParseSpec.format === 'tsv'), - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.delimiter', - type: 'string', - label: 'Delimiter', - placeholder: `(optional)`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - model.extractionNamespace.namespaceParseSpec.format === 'tsv', - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.listDelimiter', - type: 'string', - label: 'List delimiter', - placeholder: `(optional)`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - model.extractionNamespace.namespaceParseSpec.format === 'tsv', - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.keyFieldName', - type: 'string', - label: 'Key field name', - placeholder: `key`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - model.extractionNamespace.namespaceParseSpec.format === 'customJson', - ), - }, - { - name: 'extractionNamespace.namespaceParseSpec.valueFieldName', - type: 'string', - label: 'Value field name', - placeholder: `value`, - defined: (model: LookupSpec) => - Boolean( - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri' && - model.extractionNamespace.namespaceParseSpec && - model.extractionNamespace.namespaceParseSpec.format === 'customJson', - ), - }, - { - name: 'extractionNamespace.namespace', - type: 'string', - label: 'Namespace', - placeholder: 'some_lookup', - info: ( - <> -

The namespace value in the SQL query:

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.connectorConfig.createTables', - type: 'boolean', - label: 'CreateTables', - info: 'Defines the connectURI value on the The connector config to used', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.connectorConfig.connectURI', - type: 'string', - label: 'Connect URI', - info: 'Defines the connectURI value on the The connector config to used', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.connectorConfig.user', - type: 'string', - label: 'User', - info: 'Defines the user to be used by the connector config', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.connectorConfig.password', - type: 'string', - label: 'Password', - info: 'Defines the password to be used by the connector config', - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.table', - type: 'string', - label: 'Table', - placeholder: 'some_lookup_table', - info: ( - <> -

- The table which contains the key value pairs. This will become the table value in the SQL - query: -

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.keyColumn', - type: 'string', - label: 'Key column', - placeholder: 'my_key_value', - info: ( - <> -

- The column in the table which contains the keys. This will become the keyColumn value in - the SQL query: -

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.valueColumn', - type: 'string', - label: 'Value column', - placeholder: 'my_column_value', - info: ( - <> -

- The column in table which contains the values. This will become the valueColumn value in - the SQL query: -

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.filter', - type: 'string', - label: 'Filter', - placeholder: '(optional)', - info: ( - <> -

- The filter to be used when selecting lookups, this is used to create a where clause on - lookup population. This will become the expression filter in the SQL query: -

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE{' '} - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.tsColumn', - type: 'string', - label: 'TsColumn', - placeholder: '(optional)', - info: ( - <> -

- The column in table which contains when the key was updated. This will become the Value in - the SQL query: -

-

- SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE - filter -

- - ), - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'jdbc', - }, - { - name: 'extractionNamespace.pollPeriod', - type: 'string', - label: 'Poll period', - placeholder: '(optional)', - info: `Period between polling for updates`, - defined: (model: LookupSpec) => - model.type === 'cachedNamespace' && - !!model.extractionNamespace && - model.extractionNamespace.type === 'uri', - }, - { - name: 'firstCacheTimeout', - type: 'number', - label: 'First cache timeout', - placeholder: '(optional)', - info: `How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait`, - defined: (model: LookupSpec) => model.type === 'cachedNamespace', - }, - { - name: 'injective', - type: 'boolean', - defaultValue: false, - info: `If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to true`, - defined: (model: LookupSpec) => model.type === 'cachedNamespace', - }, -]; - export const LookupEditDialog = React.memo(function LookupEditDialog(props: LookupEditDialogProps) { const { onClose, @@ -565,6 +72,7 @@ export const LookupEditDialog = React.memo(function LookupEditDialog(props: Look onChange('name', e.target.value)} + intent={lookupName ? Intent.NONE : Intent.PRIMARY} disabled={isEdit} placeholder="Enter the lookup name" /> @@ -631,7 +139,7 @@ export const LookupEditDialog = React.memo(function LookupEditDialog(props: Look onClick={() => { onSubmit(updateVersionOnSubmit && isEdit); }} - disabled={isLookupSubmitDisabled(lookupName, lookupVersion, lookupTier, lookupSpec)} + disabled={isLookupInvalid(lookupName, lookupVersion, lookupTier, lookupSpec)} /> diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap index a5e42b28c7d..a4a89269264 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap @@ -11,7 +11,7 @@ exports[`overload dynamic config matches snapshot 1`] = ` Edit the overlord dynamic configuration on the fly. For more information please refer to the documentation diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx index 64cea824138..57ced4b9ac6 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx @@ -52,7 +52,7 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo processQuery: async () => { try { const configResp = await axios(`/druid/indexer/v1/worker`); - setDynamicConfig(configResp.data); + setDynamicConfig(configResp.data || {}); } catch (e) { AppToaster.show({ icon: IconNames.ERROR, diff --git a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap index 8c85f7f7f4d..380ea446001 100644 --- a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap @@ -58,7 +58,7 @@ exports[`retention dialog matches snapshot 1`] = ` Druid uses rules to determine what data should be retained in the cluster. The rules are evaluated in order from top to bottom. For more information please refer to the diff --git a/web-console/src/dialogs/spec-dialog/spec-dialog.scss b/web-console/src/dialogs/spec-dialog/spec-dialog.scss index 17d4cffb356..b8c71986ba6 100644 --- a/web-console/src/dialogs/spec-dialog/spec-dialog.scss +++ b/web-console/src/dialogs/spec-dialog/spec-dialog.scss @@ -25,9 +25,5 @@ .spec-dialog-textarea { background-color: #232c35; margin-bottom: 10px; - - .ace-solarized-dark { - background-color: #232c35; - } } } diff --git a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx index 41897f78842..e1c6b078bf4 100644 --- a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx +++ b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx @@ -21,8 +21,8 @@ import React, { useState } from 'react'; import { ShowJson } from '../../components'; import { ShowHistory } from '../../components/show-history/show-history'; import { SupervisorStatisticsTable } from '../../components/supervisor-statistics-table/supervisor-statistics-table'; +import { deepGet } from '../../utils'; import { BasicAction } from '../../utils/basic-action'; -import { deepGet } from '../../utils/object-change'; import { SideButtonMetaData, TableActionDialog } from '../table-action-dialog/table-action-dialog'; interface SupervisorTableActionDialogProps { diff --git a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx index b5f9440cc7b..d030821f8c5 100644 --- a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx +++ b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx @@ -19,8 +19,8 @@ import React, { useState } from 'react'; import { ShowJson, ShowLog } from '../../components'; +import { deepGet } from '../../utils'; import { BasicAction } from '../../utils/basic-action'; -import { deepGet } from '../../utils/object-change'; import { SideButtonMetaData, TableActionDialog } from '../table-action-dialog/table-action-dialog'; interface TaskTableActionDialogProps { diff --git a/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap b/web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap similarity index 100% rename from web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap rename to web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap diff --git a/web-console/src/druid-models/compaction-config.tsx b/web-console/src/druid-models/compaction-config.tsx new file mode 100644 index 00000000000..02bb42f0004 --- /dev/null +++ b/web-console/src/druid-models/compaction-config.tsx @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { Field } from '../components'; +import { deepGet, deepSet, oneOf } from '../utils'; + +export type CompactionConfig = Record; + +export const COMPACTION_CONFIG_FIELDS: Field[] = [ + { + name: 'skipOffsetFromLatest', + type: 'string', + defaultValue: 'P1D', + suggestions: ['PT0H', 'PT1H', 'P1D', 'P3D'], + info: ( +

+ The offset for searching segments to be compacted. Strongly recommended to set for realtime + dataSources. +

+ ), + }, + { + name: 'tuningConfig.partitionsSpec.type', + label: 'Partitioning type', + type: 'string', + suggestions: ['dynamic', 'hashed', 'single_dim'], + info: ( +

+ For perfect rollup, you should use either hashed (partitioning based on the + hash of dimensions in each row) or single_dim (based on ranges of a single + dimension). For best-effort rollup, you should use dynamic. +

+ ), + }, + // partitionsSpec type: dynamic + { + name: 'tuningConfig.partitionsSpec.maxRowsPerSegment', + label: 'Max rows per segment', + type: 'number', + defaultValue: 5000000, + defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic', + info: <>Determines how many rows are in each segment., + }, + { + name: 'tuningConfig.partitionsSpec.maxTotalRows', + label: 'Max total rows', + type: 'number', + defaultValue: 20000000, + defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic', + info: <>Total number of rows in segments waiting for being pushed., + }, + // partitionsSpec type: hashed + { + name: 'tuningConfig.partitionsSpec.targetRowsPerSegment', + label: 'Target rows per segment', + type: 'number', + zeroMeansUndefined: true, + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' && + !deepGet(t, 'tuningConfig.partitionsSpec.numShards'), + info: ( + <> +

+ If the segments generated are a sub-optimal size for the requested partition dimensions, + consider setting this field. +

+

+ A target row count for each partition. Each partition will have a row count close to the + target assuming evenly distributed keys. Defaults to 5 million if numShards is null. +

+ + ), + }, + { + name: 'tuningConfig.partitionsSpec.numShards', + label: 'Num shards', + type: 'number', + zeroMeansUndefined: true, + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' && + !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'), + info: ( + <> +

+ If you know the optimal number of shards and want to speed up the time it takes for + compaction to run, set this field. +

+

+ Directly specify the number of shards to create. If this is specified and 'intervals' is + specified in the granularitySpec, the index task can skip the determine + intervals/partitions pass through the data. +

+ + ), + }, + { + name: 'tuningConfig.partitionsSpec.partitionDimensions', + label: 'Partition dimensions', + type: 'string-array', + placeholder: '(all dimensions)', + defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed', + info:

The dimensions to partition on. Leave blank to select all dimensions.

, + }, + // partitionsSpec type: single_dim + { + name: 'tuningConfig.partitionsSpec.partitionDimension', + label: 'Partition dimension', + type: 'string', + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim', + required: true, + info:

The dimension to partition on.

, + }, + { + name: 'tuningConfig.partitionsSpec.targetRowsPerSegment', + label: 'Target rows per segment', + type: 'number', + zeroMeansUndefined: true, + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' && + !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), + required: (t: CompactionConfig) => + !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') && + !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), + info: ( +

+ Target number of rows to include in a partition, should be a number that targets segments of + 500MB~1GB. +

+ ), + }, + { + name: 'tuningConfig.partitionsSpec.maxRowsPerSegment', + label: 'Max rows per segment', + type: 'number', + zeroMeansUndefined: true, + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' && + !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'), + required: (t: CompactionConfig) => + !deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') && + !deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'), + info:

Maximum number of rows to include in a partition.

, + }, + { + name: 'tuningConfig.partitionsSpec.assumeGrouped', + label: 'Assume grouped', + type: 'boolean', + defaultValue: false, + defined: (t: CompactionConfig) => + deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim', + info: ( +

+ Assume that input data has already been grouped on time and dimensions. Ingestion will run + faster, but may choose sub-optimal partitions if this assumption is violated. +

+ ), + }, + { + name: 'tuningConfig.maxNumConcurrentSubTasks', + label: 'Max num concurrent sub tasks', + type: 'number', + defaultValue: 1, + min: 1, + info: ( + <> + Maximum number of tasks which can be run at the same time. The supervisor task would spawn + worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this + value is set to 1, the supervisor task processes data ingestion on its own instead of + spawning worker tasks. If this value is set to too large, too many worker tasks can be + created which might block other ingestion. + + ), + }, + { + name: 'inputSegmentSizeBytes', + type: 'number', + defaultValue: 419430400, + info: ( +

+ Maximum number of total segment bytes processed per compaction task. Since a time chunk must + be processed in its entirety, if the segments for a particular time chunk have a total size + in bytes greater than this parameter, compaction will not run for that time chunk. Because + each compaction task runs with a single thread, setting this value too far above 1–2GB will + result in compaction tasks taking an excessive amount of time. +

+ ), + }, + { + name: 'tuningConfig.totalNumMergeTasks', + label: 'Total num merge tasks', + type: 'number', + defaultValue: 10, + min: 1, + defined: (t: CompactionConfig) => + oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'), + info: <>Maximum number of merge tasks which can be run at the same time., + }, + { + name: 'tuningConfig.splitHintSpec.maxInputSegmentBytesPerTask', + label: 'Max input segment bytes per task', + type: 'number', + defaultValue: 500000000, + min: 1000000, + adjustment: (t: CompactionConfig) => deepSet(t, 'tuningConfig.splitHintSpec.type', 'segments'), + info: ( + <> + Maximum number of bytes of input segments to process in a single task. If a single segment + is larger than this number, it will be processed by itself in a single task (input segments + are never split across tasks). + + ), + }, +]; diff --git a/web-console/src/utils/compaction.spec.ts b/web-console/src/druid-models/compaction-status.spec.ts similarity index 95% rename from web-console/src/utils/compaction.spec.ts rename to web-console/src/druid-models/compaction-status.spec.ts index dfb1420a9d9..68b4cca339e 100644 --- a/web-console/src/utils/compaction.spec.ts +++ b/web-console/src/druid-models/compaction-status.spec.ts @@ -16,14 +16,14 @@ * limitations under the License. */ +import { CompactionConfig } from './compaction-config'; import { - CompactionConfig, CompactionStatus, formatCompactionConfigAndStatus, zeroCompactionStatus, -} from './compaction'; +} from './compaction-status'; -describe('compaction', () => { +describe('compaction status', () => { const BASIC_CONFIG: CompactionConfig = {}; const ZERO_STATUS: CompactionStatus = { dataSource: 'tbl', diff --git a/web-console/src/utils/compaction.ts b/web-console/src/druid-models/compaction-status.ts similarity index 97% rename from web-console/src/utils/compaction.ts rename to web-console/src/druid-models/compaction-status.ts index 287b8ec151e..c637e44b8a2 100644 --- a/web-console/src/utils/compaction.ts +++ b/web-console/src/druid-models/compaction-status.ts @@ -16,6 +16,8 @@ * limitations under the License. */ +import { CompactionConfig } from './compaction-config'; + function capitalizeFirst(str: string): string { return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase(); } @@ -34,8 +36,6 @@ export interface CompactionStatus { intervalCountSkipped: number; } -export type CompactionConfig = Record; - export function zeroCompactionStatus(compactionStatus: CompactionStatus): boolean { return ( !compactionStatus.bytesAwaitingCompaction && diff --git a/web-console/src/druid-models/dimension-spec.spec.ts b/web-console/src/druid-models/dimension-spec.spec.ts new file mode 100644 index 00000000000..c018c9d819d --- /dev/null +++ b/web-console/src/druid-models/dimension-spec.spec.ts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDimensionSpecs } from './dimension-spec'; + +describe('dimension-spec', () => { + it('getDimensionSpecs', () => { + expect(getDimensionSpecs({ header: ['header'], rows: [] }, {}, true)).toMatchInlineSnapshot(` + Array [ + "header", + ] + `); + }); +}); diff --git a/web-console/src/druid-models/dimension-spec.ts b/web-console/src/druid-models/dimension-spec.ts new file mode 100644 index 00000000000..e305d672cb0 --- /dev/null +++ b/web-console/src/druid-models/dimension-spec.ts @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Field } from '../components'; +import { filterMap } from '../utils'; +import { HeaderAndRows } from '../utils/sampler'; + +import { getColumnTypeFromHeaderAndRows } from './ingestion-spec'; + +export interface DimensionsSpec { + dimensions?: (string | DimensionSpec)[]; + dimensionExclusions?: string[]; + spatialDimensions?: any[]; +} + +export interface DimensionSpec { + type: string; + name: string; + createBitmapIndex?: boolean; +} + +export const DIMENSION_SPEC_FIELDS: Field[] = [ + { + name: 'name', + type: 'string', + }, + { + name: 'type', + type: 'string', + suggestions: ['string', 'long', 'float', 'double'], + }, + { + name: 'createBitmapIndex', + type: 'boolean', + defaultValue: true, + defined: (dimensionSpec: DimensionSpec) => dimensionSpec.type === 'string', + }, +]; + +export function getDimensionSpecName(dimensionSpec: string | DimensionSpec): string { + return typeof dimensionSpec === 'string' ? dimensionSpec : dimensionSpec.name; +} + +export function getDimensionSpecType(dimensionSpec: string | DimensionSpec): string { + return typeof dimensionSpec === 'string' ? 'string' : dimensionSpec.type; +} + +export function inflateDimensionSpec(dimensionSpec: string | DimensionSpec): DimensionSpec { + return typeof dimensionSpec === 'string' + ? { name: dimensionSpec, type: 'string' } + : dimensionSpec; +} + +export function getDimensionSpecs( + headerAndRows: HeaderAndRows, + typeHints: Record, + hasRollup: boolean, +): (string | DimensionSpec)[] { + return filterMap(headerAndRows.header, h => { + if (h === '__time') return; + const type = typeHints[h] || getColumnTypeFromHeaderAndRows(headerAndRows, h); + if (type === 'string') return h; + if (hasRollup) return; + return { + type, + name: h, + }; + }); +} diff --git a/web-console/src/druid-models/filter.tsx b/web-console/src/druid-models/filter.tsx new file mode 100644 index 00000000000..10791294063 --- /dev/null +++ b/web-console/src/druid-models/filter.tsx @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Field } from '../components'; +import { deepGet, EMPTY_ARRAY, oneOf } from '../utils'; + +export type DruidFilter = Record; + +export interface DimensionFiltersWithRest { + dimensionFilters: DruidFilter[]; + restFilter?: DruidFilter; +} + +export function splitFilter(filter: DruidFilter | null): DimensionFiltersWithRest { + const inputAndFilters: DruidFilter[] = filter + ? filter.type === 'and' && Array.isArray(filter.fields) + ? filter.fields + : [filter] + : EMPTY_ARRAY; + const dimensionFilters: DruidFilter[] = inputAndFilters.filter( + f => typeof f.dimension === 'string', + ); + const restFilters: DruidFilter[] = inputAndFilters.filter(f => typeof f.dimension !== 'string'); + + return { + dimensionFilters, + restFilter: restFilters.length + ? restFilters.length > 1 + ? { type: 'and', filters: restFilters } + : restFilters[0] + : undefined, + }; +} + +export function joinFilter( + dimensionFiltersWithRest: DimensionFiltersWithRest, +): DruidFilter | undefined { + const { dimensionFilters, restFilter } = dimensionFiltersWithRest; + let newFields = dimensionFilters || EMPTY_ARRAY; + if (restFilter && restFilter.type) newFields = newFields.concat([restFilter]); + + if (!newFields.length) return; + if (newFields.length === 1) return newFields[0]; + return { type: 'and', fields: newFields }; +} + +export const FILTER_FIELDS: Field[] = [ + { + name: 'type', + type: 'string', + suggestions: ['selector', 'in', 'regex', 'like', 'not'], + }, + { + name: 'dimension', + type: 'string', + defined: (df: DruidFilter) => oneOf(df.type, 'selector', 'in', 'regex', 'like'), + }, + { + name: 'value', + type: 'string', + defined: (df: DruidFilter) => df.type === 'selector', + }, + { + name: 'values', + type: 'string-array', + defined: (df: DruidFilter) => df.type === 'in', + }, + { + name: 'pattern', + type: 'string', + defined: (df: DruidFilter) => oneOf(df.type, 'regex', 'like'), + }, + + { + name: 'field.type', + label: 'Sub-filter type', + type: 'string', + suggestions: ['selector', 'in', 'regex', 'like'], + defined: (df: DruidFilter) => df.type === 'not', + }, + { + name: 'field.dimension', + label: 'Sub-filter dimension', + type: 'string', + defined: (df: DruidFilter) => df.type === 'not', + }, + { + name: 'field.value', + label: 'Sub-filter value', + type: 'string', + defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'selector', + }, + { + name: 'field.values', + label: 'Sub-filter values', + type: 'string-array', + defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'in', + }, + { + name: 'field.pattern', + label: 'Sub-filter pattern', + type: 'string', + defined: (df: DruidFilter) => + df.type === 'not' && oneOf(deepGet(df, 'field.type'), 'regex', 'like'), + }, +]; diff --git a/web-console/src/utils/spec-utils.spec.ts b/web-console/src/druid-models/flatten-spec.spec.ts similarity index 97% rename from web-console/src/utils/spec-utils.spec.ts rename to web-console/src/druid-models/flatten-spec.spec.ts index f3684173b76..faf8987a255 100644 --- a/web-console/src/utils/spec-utils.spec.ts +++ b/web-console/src/druid-models/flatten-spec.spec.ts @@ -16,7 +16,7 @@ * limitations under the License. */ -import { computeFlattenExprsForData } from './spec-utils'; +import { computeFlattenExprsForData } from './flatten-spec'; describe('spec-utils', () => { describe('computeFlattenExprsForData', () => { diff --git a/web-console/src/utils/spec-utils.ts b/web-console/src/druid-models/flatten-spec.tsx similarity index 73% rename from web-console/src/utils/spec-utils.ts rename to web-console/src/druid-models/flatten-spec.tsx index 2842118ba99..f4246f1df2e 100644 --- a/web-console/src/utils/spec-utils.ts +++ b/web-console/src/druid-models/flatten-spec.tsx @@ -16,7 +16,50 @@ * limitations under the License. */ -import { FlattenField } from './ingestion-spec'; +import React from 'react'; + +import { ExternalLink, Field } from '../components'; +import { getLink } from '../links'; +import { oneOf } from '../utils'; + +export interface FlattenSpec { + useFieldDiscovery?: boolean; + fields?: FlattenField[]; +} + +export interface FlattenField { + name: string; + type: string; + expr: string; +} + +export const FLATTEN_FIELD_FIELDS: Field[] = [ + { + name: 'name', + type: 'string', + placeholder: 'column_name', + required: true, + }, + { + name: 'type', + type: 'string', + suggestions: ['path', 'jq', 'root'], + required: true, + }, + { + name: 'expr', + type: 'string', + placeholder: '$.thing', + defined: (flattenField: FlattenField) => oneOf(flattenField.type, 'path', 'jq'), + required: true, + info: ( + <> + Specify a flatten{' '} + expression. + + ), + }, +]; export type ExprType = 'path' | 'jq'; export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts new file mode 100644 index 00000000000..5e7debe93cf --- /dev/null +++ b/web-console/src/druid-models/index.ts @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './compaction-config'; +export * from './compaction-status'; +export * from './lookup-spec'; +export * from './time'; +export * from './timestamp-spec'; +export * from './transform-spec'; +export * from './input-source'; +export * from './input-format'; +export * from './flatten-spec'; +export * from './filter'; +export * from './dimension-spec'; +export * from './metric-spec'; +export * from './ingestion-spec'; diff --git a/web-console/src/utils/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec.spec.ts similarity index 60% rename from web-console/src/utils/ingestion-spec.spec.ts rename to web-console/src/druid-models/ingestion-spec.spec.ts index 3577819b8cb..adccec656f8 100644 --- a/web-console/src/utils/ingestion-spec.spec.ts +++ b/web-console/src/druid-models/ingestion-spec.spec.ts @@ -16,7 +16,16 @@ * limitations under the License. */ -import { cleanSpec, downgradeSpec, guessInputFormat, upgradeSpec } from './ingestion-spec'; +import { + cleanSpec, + downgradeSpec, + getColumnTypeFromHeaderAndRows, + guessInputFormat, + guessTypeFromSample, + IngestionSpec, + updateSchemaWithSample, + upgradeSpec, +} from './ingestion-spec'; describe('ingestion-spec', () => { const oldSpec = { @@ -152,3 +161,98 @@ describe('ingestion-spec', () => { }); }); }); + +describe('spec utils', () => { + const ingestionSpec: IngestionSpec = { + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + tuningConfig: { + type: 'index_parallel', + }, + dataSchema: { + dataSource: 'wikipedia', + granularitySpec: { + type: 'uniform', + segmentGranularity: 'DAY', + queryGranularity: 'HOUR', + }, + timestampSpec: { + column: 'timestamp', + format: 'iso', + }, + dimensionsSpec: {}, + }, + }, + }; + + it('guessTypeFromSample', () => { + expect(guessTypeFromSample([])).toMatchInlineSnapshot(`"string"`); + }); + + it('getColumnTypeFromHeaderAndRows', () => { + expect( + getColumnTypeFromHeaderAndRows({ header: ['header'], rows: [] }, 'header'), + ).toMatchInlineSnapshot(`"string"`); + }); + + it('updateSchemaWithSample', () => { + expect( + updateSchemaWithSample(ingestionSpec, { header: ['header'], rows: [] }, 'specific', true), + ).toMatchInlineSnapshot(` + Object { + "spec": Object { + "dataSchema": Object { + "dataSource": "wikipedia", + "dimensionsSpec": Object { + "dimensions": Array [ + "header", + ], + }, + "granularitySpec": Object { + "queryGranularity": "HOUR", + "rollup": true, + "segmentGranularity": "DAY", + "type": "uniform", + }, + "metricsSpec": Array [ + Object { + "name": "count", + "type": "count", + }, + ], + "timestampSpec": Object { + "column": "timestamp", + "format": "iso", + }, + }, + "ioConfig": Object { + "inputFormat": Object { + "type": "json", + }, + "inputSource": Object { + "type": "http", + "uris": Array [ + "https://static.imply.io/data/wikipedia.json.gz", + ], + }, + "type": "index_parallel", + }, + "tuningConfig": Object { + "type": "index_parallel", + }, + }, + "type": "index_parallel", + } + `); + }); +}); diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx similarity index 72% rename from web-console/src/utils/ingestion-spec.tsx rename to web-console/src/druid-models/ingestion-spec.tsx index 73f6d43297f..e58c5916504 100644 --- a/web-console/src/utils/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec.tsx @@ -19,24 +19,39 @@ import { Code } from '@blueprintjs/core'; import React from 'react'; -import { Field } from '../components/auto-form/auto-form'; -import { ExternalLink } from '../components/external-link/external-link'; +import { ExternalLink, Field } from '../components'; import { getLink } from '../links'; +import { + deepDelete, + deepGet, + deepMove, + deepSet, + EMPTY_ARRAY, + EMPTY_OBJECT, + filterMap, + oneOf, +} from '../utils'; +import { HeaderAndRows } from '../utils/sampler'; import { - BASIC_TIME_FORMATS, - DATE_ONLY_TIME_FORMATS, - DATETIME_TIME_FORMATS, - OTHER_TIME_FORMATS, -} from './druid-time'; -import { deepDelete, deepGet, deepMove, deepSet } from './object-change'; + DimensionsSpec, + getDimensionSpecName, + getDimensionSpecs, + getDimensionSpecType, +} from './dimension-spec'; +import { InputFormat, issueWithInputFormat } from './input-format'; +import { InputSource, issueWithInputSource } from './input-source'; +import { + getMetricSpecOutputType, + getMetricSpecs, + getMetricSpecSingleFieldName, + MetricSpec, +} from './metric-spec'; +import { PLACEHOLDER_TIMESTAMP_SPEC, TimestampSpec } from './timestamp-spec'; +import { TransformSpec } from './transform-spec'; export const MAX_INLINE_DATA_LENGTH = 65536; -// These constants are used to make sure that they are not constantly recreated thrashing the pure components -export const EMPTY_OBJECT: any = {}; -export const EMPTY_ARRAY: any[] = []; - const CURRENT_YEAR = new Date().getUTCFullYear(); export interface IngestionSpec { @@ -77,14 +92,6 @@ export type IngestionComboTypeWithExtra = | 'example' | 'other'; -export function adjustIngestionSpec(spec: IngestionSpec) { - const tuningConfig = deepGet(spec, 'spec.tuningConfig'); - if (tuningConfig) { - spec = deepSet(spec, 'spec.tuningConfig', adjustTuningConfig(tuningConfig)); - } - return spec; -} - function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): string { switch (ingestionType) { case 'kafka': @@ -189,7 +196,7 @@ export function getIngestionDocLink(spec: IngestionSpec): string { return `${getLink('DOCS')}/development/extensions-core/kinesis-ingestion.html`; default: - return `${getLink('DOCS')}/ingestion/native-batch.html#firehoses`; + return `${getLink('DOCS')}/ingestion/native-batch.html#input-sources`; } } @@ -229,18 +236,6 @@ export interface DataSchema { metricsSpec?: MetricSpec[]; } -export interface InputFormat { - type: string; - findColumnsFromHeader?: boolean; - skipHeaderRows?: number; - columns?: string[]; - listDelimiter?: string; - pattern?: string; - function?: string; - flattenSpec?: FlattenSpec; - keepNullColumns?: boolean; -} - export type DimensionMode = 'specific' | 'auto-detect'; export function getDimensionMode(spec: IngestionSpec): DimensionMode { @@ -266,7 +261,7 @@ export function isTask(spec: IngestionSpec) { const type = String(getSpecType(spec)); return ( type.startsWith('index_') || - ['index', 'compact', 'kill', 'append', 'merge', 'same_interval_merge'].includes(type) + oneOf(type, 'index', 'compact', 'kill', 'append', 'merge', 'same_interval_merge') ); } @@ -314,359 +309,6 @@ export function cleanSpec(spec: IngestionSpec): IngestionSpec { }; } -const INPUT_FORMAT_FORM_FIELDS: Field[] = [ - { - name: 'type', - label: 'Input format', - type: 'string', - suggestions: ['json', 'csv', 'tsv', 'regex', 'parquet', 'orc', 'avro_ocf'], - info: ( - <> -

The parser used to parse the data.

-

- For more information see{' '} - - the documentation - - . -

- - ), - }, - { - name: 'pattern', - type: 'string', - required: true, - defined: (p: InputFormat) => p.type === 'regex', - }, - { - name: 'function', - type: 'string', - required: true, - defined: (p: InputFormat) => p.type === 'javascript', - }, - { - name: 'findColumnsFromHeader', - type: 'boolean', - required: true, - defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv', - }, - { - name: 'skipHeaderRows', - type: 'number', - defaultValue: 0, - defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv', - min: 0, - info: ( - <> - If both skipHeaderRows and hasHeaderRow options are set, skipHeaderRows is first applied. - For example, if you set skipHeaderRows to 2 and hasHeaderRow to true, Druid will skip the - first two lines and then extract column information from the third line. - - ), - }, - { - name: 'columns', - type: 'string-array', - required: (p: InputFormat) => - ((p.type === 'csv' || p.type === 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex', - defined: (p: InputFormat) => - ((p.type === 'csv' || p.type === 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex', - }, - { - name: 'delimiter', - type: 'string', - defaultValue: '\t', - defined: (p: InputFormat) => p.type === 'tsv', - info: <>A custom delimiter for data values., - }, - { - name: 'listDelimiter', - type: 'string', - defined: (p: InputFormat) => p.type === 'csv' || p.type === 'tsv' || p.type === 'regex', - info: <>A custom delimiter for multi-value dimensions., - }, - { - name: 'binaryAsString', - type: 'boolean', - defaultValue: false, - defined: (p: InputFormat) => p.type === 'parquet' || p.type === 'orc' || p.type === 'avro_ocf', - info: ( - <> - Specifies if the bytes parquet column which is not logically marked as a string or enum type - should be treated as a UTF-8 encoded string. - - ), - }, -]; - -export function getInputFormatFormFields() { - return INPUT_FORMAT_FORM_FIELDS; -} - -export function issueWithInputFormat(inputFormat: InputFormat | undefined): string | undefined { - if (!inputFormat) return 'no input format'; - if (!inputFormat.type) return 'missing a type'; - switch (inputFormat.type) { - case 'regex': - if (!inputFormat.pattern) return "must have a 'pattern'"; - break; - - case 'javascript': - if (!inputFormat['function']) return "must have a 'function'"; - break; - } - return; -} - -export function inputFormatCanFlatten(inputFormat: InputFormat): boolean { - const inputFormatType = inputFormat.type; - return ( - inputFormatType === 'json' || - inputFormatType === 'parquet' || - inputFormatType === 'orc' || - inputFormatType === 'avro_ocf' - ); -} - -export interface TimestampSpec { - column?: string; - format?: string; - missingValue?: string; -} - -export function getTimestampSpecColumn(timestampSpec: TimestampSpec) { - // https://github.com/apache/druid/blob/master/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java#L44 - return timestampSpec.column || 'timestamp'; -} - -const NO_SUCH_COLUMN = '!!!_no_such_column_!!!'; - -const DUMMY_TIMESTAMP_SPEC: TimestampSpec = { - column: NO_SUCH_COLUMN, - missingValue: '1970-01-01T00:00:00Z', -}; - -export function getDummyTimestampSpec() { - return DUMMY_TIMESTAMP_SPEC; -} - -const CONSTANT_TIMESTAMP_SPEC: TimestampSpec = { - column: NO_SUCH_COLUMN, - missingValue: '2010-01-01T00:00:00Z', -}; - -export function getConstantTimestampSpec() { - return CONSTANT_TIMESTAMP_SPEC; -} - -export function isColumnTimestampSpec(timestampSpec: TimestampSpec) { - return (deepGet(timestampSpec, 'column') || 'timestamp') !== NO_SUCH_COLUMN; -} - -const TIMESTAMP_SPEC_FORM_FIELDS: Field[] = [ - { - name: 'column', - type: 'string', - defaultValue: 'timestamp', - }, - { - name: 'format', - type: 'string', - defaultValue: 'auto', - suggestions: [ - ...BASIC_TIME_FORMATS, - { - group: 'Date and time formats', - suggestions: DATETIME_TIME_FORMATS, - }, - { - group: 'Date only formats', - suggestions: DATE_ONLY_TIME_FORMATS, - }, - { - group: 'Other time formats', - suggestions: OTHER_TIME_FORMATS, - }, - ], - defined: (timestampSpec: TimestampSpec) => isColumnTimestampSpec(timestampSpec), - info: ( -

- Please specify your timestamp format by using the suggestions menu or typing in a{' '} - - format string - - . -

- ), - }, - { - name: 'missingValue', - type: 'string', - placeholder: '(optional)', - info:

This value will be used if the specified column can not be found.

, - }, -]; - -const CONSTANT_TIMESTAMP_SPEC_FORM_FIELDS: Field[] = [ - { - name: 'missingValue', - label: 'Constant value', - type: 'string', - info:

The dummy value that will be used as the timestamp.

, - }, -]; - -export function getTimestampSpecFormFields(timestampSpec: TimestampSpec) { - if (isColumnTimestampSpec(timestampSpec)) { - return TIMESTAMP_SPEC_FORM_FIELDS; - } else { - return CONSTANT_TIMESTAMP_SPEC_FORM_FIELDS; - } -} - -export function issueWithTimestampSpec( - timestampSpec: TimestampSpec | undefined, -): string | undefined { - if (!timestampSpec) return 'no spec'; - if (!timestampSpec.column && !timestampSpec.missingValue) return 'timestamp spec is blank'; - return; -} - -export interface DimensionsSpec { - dimensions?: (string | DimensionSpec)[]; - dimensionExclusions?: string[]; - spatialDimensions?: any[]; -} - -export interface DimensionSpec { - type: string; - name: string; - createBitmapIndex?: boolean; -} - -const DIMENSION_SPEC_FORM_FIELDS: Field[] = [ - { - name: 'name', - type: 'string', - }, - { - name: 'type', - type: 'string', - suggestions: ['string', 'long', 'float', 'double'], - }, - { - name: 'createBitmapIndex', - type: 'boolean', - defaultValue: true, - defined: (dimensionSpec: DimensionSpec) => dimensionSpec.type === 'string', - }, -]; - -export function getDimensionSpecFormFields() { - return DIMENSION_SPEC_FORM_FIELDS; -} - -export function getDimensionSpecName(dimensionSpec: string | DimensionSpec): string { - return typeof dimensionSpec === 'string' ? dimensionSpec : dimensionSpec.name; -} - -export function getDimensionSpecType(dimensionSpec: string | DimensionSpec): string { - return typeof dimensionSpec === 'string' ? 'string' : dimensionSpec.type; -} - -export function inflateDimensionSpec(dimensionSpec: string | DimensionSpec): DimensionSpec { - return typeof dimensionSpec === 'string' - ? { name: dimensionSpec, type: 'string' } - : dimensionSpec; -} - -export interface FlattenSpec { - useFieldDiscovery?: boolean; - fields?: FlattenField[]; -} - -export interface FlattenField { - name: string; - type: string; - expr: string; -} - -const FLATTEN_FIELD_FORM_FIELDS: Field[] = [ - { - name: 'name', - type: 'string', - placeholder: 'column_name', - required: true, - }, - { - name: 'type', - type: 'string', - suggestions: ['path', 'jq', 'root'], - required: true, - }, - { - name: 'expr', - type: 'string', - placeholder: '$.thing', - defined: (flattenField: FlattenField) => - flattenField.type === 'path' || flattenField.type === 'jq', - required: true, - info: ( - <> - Specify a flatten{' '} - expression. - - ), - }, -]; - -export function getFlattenFieldFormFields() { - return FLATTEN_FIELD_FORM_FIELDS; -} - -export interface TransformSpec { - transforms?: Transform[]; - filter?: any; -} - -export interface Transform { - type: string; - name: string; - expression: string; -} - -const TRANSFORM_FORM_FIELDS: Field[] = [ - { - name: 'name', - type: 'string', - placeholder: 'output_name', - required: true, - }, - { - name: 'type', - type: 'string', - suggestions: ['expression'], - required: true, - }, - { - name: 'expression', - type: 'string', - placeholder: '"foo" + "bar"', - required: true, - info: ( - <> - A valid Druid{' '} - expression. - - ), - }, -]; - -export function getTransformFormFields() { - return TRANSFORM_FORM_FIELDS; -} - export interface GranularitySpec { type?: string; queryGranularity?: string; @@ -675,294 +317,6 @@ export interface GranularitySpec { intervals?: string | string[]; } -export interface MetricSpec { - type: string; - name?: string; - fieldName?: string; - maxStringBytes?: number; - filterNullValues?: boolean; - fieldNames?: string[]; - fnAggregate?: string; - fnCombine?: string; - fnReset?: string; - fields?: string[]; - byRow?: boolean; - round?: boolean; - isInputHyperUnique?: boolean; - filter?: any; - aggregator?: MetricSpec; -} - -const METRIC_SPEC_FORM_FIELDS: Field[] = [ - { - name: 'name', - type: 'string', - info: <>The metric name as it will appear in Druid., - }, - { - name: 'type', - type: 'string', - suggestions: [ - 'count', - { - group: 'sum', - suggestions: ['longSum', 'doubleSum', 'floatSum'], - }, - { - group: 'min', - suggestions: ['longMin', 'doubleMin', 'floatMin'], - }, - { - group: 'max', - suggestions: ['longMax', 'doubleMax', 'floatMax'], - }, - { - group: 'first', - suggestions: ['longFirst', 'doubleFirst', 'floatFirst'], - }, - { - group: 'last', - suggestions: ['longLast', 'doubleLast', 'floatLast'], - }, - 'thetaSketch', - { - group: 'HLLSketch', - suggestions: ['HLLSketchBuild', 'HLLSketchMerge'], - }, - 'quantilesDoublesSketch', - 'momentSketch', - 'fixedBucketsHistogram', - 'hyperUnique', - 'filtered', - ], - info: <>The aggregation function to apply., - }, - { - name: 'fieldName', - type: 'string', - defined: m => m.type !== 'filtered', - info: <>The column name for the aggregator to operate on., - }, - { - name: 'maxStringBytes', - type: 'number', - defaultValue: 1024, - defined: m => { - return ['stringFirst', 'stringLast'].includes(m.type); - }, - }, - { - name: 'filterNullValues', - type: 'boolean', - defaultValue: false, - defined: m => { - return ['stringFirst', 'stringLast'].includes(m.type); - }, - }, - // filtered - { - name: 'filter', - type: 'json', - defined: m => m.type === 'filtered', - }, - { - name: 'aggregator', - type: 'json', - defined: m => m.type === 'filtered', - }, - // thetaSketch - { - name: 'size', - type: 'number', - defined: m => m.type === 'thetaSketch', - defaultValue: 16384, - info: ( - <> -

- Must be a power of 2. Internally, size refers to the maximum number of entries sketch - object will retain. Higher size means higher accuracy but more space to store sketches. - Note that after you index with a particular size, druid will persist sketch in segments - and you will use size greater or equal to that at query time. -

-

- See the{' '} - - DataSketches site - {' '} - for details. -

-

In general, We recommend just sticking to default size.

- - ), - }, - { - name: 'isInputThetaSketch', - type: 'boolean', - defined: m => m.type === 'thetaSketch', - defaultValue: false, - info: ( - <> - This should only be used at indexing time if your input data contains theta sketch objects. - This would be the case if you use datasketches library outside of Druid, say with Pig/Hive, - to produce the data that you are ingesting into Druid - - ), - }, - // HLLSketchBuild & HLLSketchMerge - { - name: 'lgK', - type: 'number', - defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge', - defaultValue: 12, - info: ( - <> -

- log2 of K that is the number of buckets in the sketch, parameter that controls the size - and the accuracy. -

-

Must be between 4 to 21 inclusively.

- - ), - }, - { - name: 'tgtHllType', - type: 'string', - defined: m => m.type === 'HLLSketchBuild' || m.type === 'HLLSketchMerge', - defaultValue: 'HLL_4', - suggestions: ['HLL_4', 'HLL_6', 'HLL_8'], - info: ( - <> - The type of the target HLL sketch. Must be HLL_4, HLL_6, or{' '} - HLL_8. - - ), - }, - // quantilesDoublesSketch - { - name: 'k', - type: 'number', - defined: m => m.type === 'quantilesDoublesSketch', - defaultValue: 128, - info: ( - <> -

- Parameter that determines the accuracy and size of the sketch. Higher k means higher - accuracy but more space to store sketches. -

-

- Must be a power of 2 from 2 to 32768. See the{' '} - - Quantiles Accuracy - {' '} - for details. -

- - ), - }, - // momentSketch - { - name: 'k', - type: 'number', - defined: m => m.type === 'momentSketch', - required: true, - info: ( - <> - Parameter that determines the accuracy and size of the sketch. Higher k means higher - accuracy but more space to store sketches. Usable range is generally [3,15] - - ), - }, - { - name: 'compress', - type: 'boolean', - defined: m => m.type === 'momentSketch', - defaultValue: true, - info: ( - <> - Flag for whether the aggregator compresses numeric values using arcsinh. Can improve - robustness to skewed and long-tailed distributions, but reduces accuracy slightly on more - uniform distributions. - - ), - }, - // fixedBucketsHistogram - { - name: 'lowerLimit', - type: 'number', - defined: m => m.type === 'fixedBucketsHistogram', - required: true, - info: <>Lower limit of the histogram., - }, - { - name: 'upperLimit', - type: 'number', - defined: m => m.type === 'fixedBucketsHistogram', - required: true, - info: <>Upper limit of the histogram., - }, - { - name: 'numBuckets', - type: 'number', - defined: m => m.type === 'fixedBucketsHistogram', - defaultValue: 10, - required: true, - info: ( - <> - Number of buckets for the histogram. The range [lowerLimit, upperLimit] will be - divided into numBuckets intervals of equal size. - - ), - }, - { - name: 'outlierHandlingMode', - type: 'string', - defined: m => m.type === 'fixedBucketsHistogram', - required: true, - suggestions: ['ignore', 'overflow', 'clip'], - info: ( - <> -

- Specifies how values outside of [lowerLimit, upperLimit] will be handled. -

-

- Supported modes are ignore, overflow, and clip. See - - outlier handling modes - {' '} - for more details. -

- - ), - }, - // hyperUnique - { - name: 'isInputHyperUnique', - type: 'boolean', - defined: m => m.type === 'hyperUnique', - defaultValue: false, - info: ( - <> - This can be set to true to index precomputed HLL (Base64 encoded output from druid-hll is - expected). - - ), - }, -]; - -export function getMetricSpecFormFields() { - return METRIC_SPEC_FORM_FIELDS; -} - -export function getMetricSpecName(metricSpec: MetricSpec): string { - return ( - metricSpec.name || (metricSpec.aggregator ? getMetricSpecName(metricSpec.aggregator) : '?') - ); -} - // -------------- export interface IoConfig { @@ -990,29 +344,6 @@ export function invalidIoConfig(ioConfig: IoConfig): boolean { ); } -export interface InputSource { - type: string; - baseDir?: string; - filter?: any; - uris?: string[]; - prefixes?: string[]; - objects?: { bucket: string; path: string }[]; - fetchTimeout?: number; - - // druid - dataSource?: string; - interval?: string; - dimensions?: string[]; - metrics?: string[]; - maxInputSegmentBytesPerTask?: number; - - // inline - data?: string; - - // hdfs - paths?: string; -} - export function getIoConfigFormFields(ingestionComboType: IngestionComboType): Field[] { const inputSourceType: Field = { name: 'inputSource.type', @@ -1022,7 +353,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F info: (

Druid connects to raw data through{' '} - + inputSources . You can change your selected inputSource here. @@ -1075,7 +406,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F required: true, info: ( <> - + inputSource.baseDir

Specifies the directory to search recursively for files to be ingested.

@@ -1099,7 +430,9 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F ], info: ( <> - + inputSource.filter

@@ -1588,55 +921,6 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F throw new Error(`unknown input type ${ingestionComboType}`); } -function nonEmptyArray(a: any) { - return Array.isArray(a) && Boolean(a.length); -} - -function issueWithInputSource(inputSource: InputSource | undefined): string | undefined { - if (!inputSource) return 'does not exist'; - if (!inputSource.type) return 'missing a type'; - switch (inputSource.type) { - case 'local': - if (!inputSource.baseDir) return `must have a 'baseDir'`; - if (!inputSource.filter) return `must have a 'filter'`; - break; - - case 'http': - if (!nonEmptyArray(inputSource.uris)) { - return 'must have at least one uri'; - } - break; - - case 'druid': - if (!inputSource.dataSource) return `must have a 'dataSource'`; - if (!inputSource.interval) return `must have an 'interval'`; - break; - - case 'inline': - if (!inputSource.data) return `must have 'data'`; - break; - - case 's3': - case 'azure': - case 'google': - if ( - !nonEmptyArray(inputSource.uris) && - !nonEmptyArray(inputSource.prefixes) && - !nonEmptyArray(inputSource.objects) - ) { - return 'must have at least one uri or prefix or object'; - } - break; - - case 'hdfs': - if (!inputSource.paths) { - return 'must have paths'; - } - break; - } - return; -} - export function issueWithIoConfig( ioConfig: IoConfig | undefined, ignoreInputFormat = false, @@ -2096,30 +1380,29 @@ export function adjustTuningConfig(tuningConfig: TuningConfig) { const tuningConfigType = deepGet(tuningConfig, 'type'); if (tuningConfigType !== 'index_parallel') return tuningConfig; - const partitionsSpecType = deepGet(tuningConfig, 'partitionsSpec.type'); - if (tuningConfig.forceGuaranteedRollup) { - if (partitionsSpecType !== 'hashed' && partitionsSpecType !== 'single_dim') { - tuningConfig = deepSet(tuningConfig, 'partitionsSpec', { type: 'hashed' }); - } - } else { - if (partitionsSpecType !== 'dynamic') { - tuningConfig = deepSet(tuningConfig, 'partitionsSpec', { type: 'dynamic' }); - } + const partitionsSpecType = deepGet(tuningConfig, 'partitionsSpec.type') || 'dynamic'; + if (partitionsSpecType === 'dynamic') { + tuningConfig = deepDelete(tuningConfig, 'forceGuaranteedRollup'); + } else if (oneOf(partitionsSpecType, 'hashed', 'single_dim')) { + tuningConfig = deepSet(tuningConfig, 'forceGuaranteedRollup', true); } + return tuningConfig; } export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any): boolean { - if (tuningConfig.type !== 'index_parallel' || !tuningConfig.forceGuaranteedRollup) return false; + if (tuningConfig.type !== 'index_parallel') return false; - if (!intervals) return true; switch (deepGet(tuningConfig, 'partitionsSpec.type')) { case 'hashed': + if (!intervals) return true; return ( Boolean(deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment')) && Boolean(deepGet(tuningConfig, 'partitionsSpec.numShards')) ); + case 'single_dim': + if (!intervals) return true; if (!deepGet(tuningConfig, 'partitionsSpec.partitionDimension')) return true; const hasTargetRowsPerSegment = Boolean( deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment'), @@ -2141,25 +1424,12 @@ export function getPartitionRelatedTuningSpecFormFields( switch (specType) { case 'index_parallel': return [ - { - name: 'forceGuaranteedRollup', - type: 'boolean', - defaultValue: false, - info: ( -

- Forces guaranteeing the perfect rollup. The perfect rollup optimizes the total size of - generated segments and querying time while indexing time will be increased. If this is - set to true, the index task will read the entire input data twice: one for finding the - optimal number of partitions per time chunk and one for generating segments. -

- ), - }, { name: 'partitionsSpec.type', label: 'Partitioning type', type: 'string', - suggestions: (t: TuningConfig) => - t.forceGuaranteedRollup ? ['hashed', 'single_dim'] : ['dynamic'], + required: true, + suggestions: ['dynamic', 'hashed', 'single_dim'], info: (

For perfect rollup, you should use either hashed (partitioning based on @@ -2355,17 +1625,26 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ info: <>Used in determining when intermediate persists to disk should occur., }, { - name: 'maxNumMergeTasks', + name: 'totalNumMergeTasks', type: 'number', defaultValue: 10, - defined: (t: TuningConfig) => Boolean(t.type === 'index_parallel' && t.forceGuaranteedRollup), + min: 1, + defined: (t: TuningConfig) => + Boolean( + t.type === 'index_parallel' && + oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'), + ), info: <>Number of tasks to merge partial segments after shuffle., }, { name: 'maxNumSegmentsToMerge', type: 'number', defaultValue: 100, - defined: (t: TuningConfig) => Boolean(t.type === 'index_parallel' && t.forceGuaranteedRollup), + defined: (t: TuningConfig) => + Boolean( + t.type === 'index_parallel' && + oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'), + ), info: ( <> Max limit for the number of segments a single task can merge at the same time after shuffle. @@ -2376,7 +1655,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'resetOffsetAutomatically', type: 'boolean', defaultValue: false, - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: ( <> Whether to reset the consumer offset if the next offset that it is trying to fetch is less @@ -2388,14 +1667,14 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'intermediatePersistPeriod', type: 'duration', defaultValue: 'PT10M', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: <>The period that determines the rate at which intermediate persists occur., }, { name: 'intermediateHandoffPeriod', type: 'duration', defaultValue: 'P2147483647D', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: ( <> How often the tasks should hand off segments. Handoff will happen either if @@ -2429,7 +1708,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'handoffConditionTimeout', type: 'number', defaultValue: 0, - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: <>Milliseconds to wait for segment handoff. 0 means to wait forever., }, { @@ -2489,7 +1768,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'workerThreads', type: 'number', placeholder: 'min(10, taskCount)', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: ( <>The number of threads that will be used by the supervisor for asynchronous operations. ), @@ -2498,14 +1777,14 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'chatThreads', type: 'number', placeholder: 'min(10, taskCount * replicas)', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: <>The number of threads that will be used for communicating with indexing tasks., }, { name: 'chatRetries', type: 'number', defaultValue: 8, - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: ( <> The number of times HTTP requests to indexing tasks will be retried before considering tasks @@ -2517,14 +1796,14 @@ const TUNING_CONFIG_FORM_FIELDS: Field[] = [ name: 'httpTimeout', type: 'duration', defaultValue: 'PT10S', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: <>How long to wait for a HTTP response from an indexing task., }, { name: 'shutdownTimeout', type: 'duration', defaultValue: 'PT80S', - defined: (t: TuningConfig) => t.type === 'kafka' || t.type === 'kinesis', + defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'), info: ( <> How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting. @@ -2676,7 +1955,7 @@ export function updateIngestionType( } if (!deepGet(spec, 'spec.dataSchema.timestampSpec')) { - newSpec = deepSet(newSpec, 'spec.dataSchema.timestampSpec', getDummyTimestampSpec()); + newSpec = deepSet(newSpec, 'spec.dataSchema.timestampSpec', PLACEHOLDER_TIMESTAMP_SPEC); } if (!deepGet(spec, 'spec.dataSchema.dimensionsSpec')) { @@ -2744,111 +2023,91 @@ function inputFormatFromType(type: string, findColumnsFromHeader?: boolean): Inp return inputFormat; } -export type DruidFilter = Record; +// ------------------------ -export interface DimensionFiltersWithRest { - dimensionFilters: DruidFilter[]; - restFilter?: DruidFilter; +export function guessTypeFromSample(sample: any[]): string { + const definedValues = sample.filter(v => v != null); + if ( + definedValues.length && + definedValues.every(v => !isNaN(v) && oneOf(typeof v, 'number', 'string')) + ) { + if (definedValues.every(v => v % 1 === 0)) { + return 'long'; + } else { + return 'double'; + } + } else { + return 'string'; + } } -export function splitFilter(filter: DruidFilter | null): DimensionFiltersWithRest { - const inputAndFilters: DruidFilter[] = filter - ? filter.type === 'and' && Array.isArray(filter.fields) - ? filter.fields - : [filter] - : EMPTY_ARRAY; - const dimensionFilters: DruidFilter[] = inputAndFilters.filter( - f => typeof f.dimension === 'string', +export function getColumnTypeFromHeaderAndRows( + headerAndRows: HeaderAndRows, + column: string, +): string { + return guessTypeFromSample( + filterMap(headerAndRows.rows, (r: any) => (r.parsed ? r.parsed[column] : undefined)), ); - const restFilters: DruidFilter[] = inputAndFilters.filter(f => typeof f.dimension !== 'string'); - - return { - dimensionFilters, - restFilter: restFilters.length - ? restFilters.length > 1 - ? { type: 'and', filters: restFilters } - : restFilters[0] - : undefined, - }; } -export function joinFilter( - dimensionFiltersWithRest: DimensionFiltersWithRest, -): DruidFilter | undefined { - const { dimensionFilters, restFilter } = dimensionFiltersWithRest; - let newFields = dimensionFilters || EMPTY_ARRAY; - if (restFilter && restFilter.type) newFields = newFields.concat([restFilter]); +function getTypeHintsFromSpec(spec: IngestionSpec): Record { + const typeHints: Record = {}; + const currentDimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions') || []; + for (const currentDimension of currentDimensions) { + typeHints[getDimensionSpecName(currentDimension)] = getDimensionSpecType(currentDimension); + } - if (!newFields.length) return; - if (newFields.length === 1) return newFields[0]; - return { type: 'and', fields: newFields }; + const currentMetrics = deepGet(spec, 'spec.dataSchema.metricsSpec') || []; + for (const currentMetric of currentMetrics) { + const singleFieldName = getMetricSpecSingleFieldName(currentMetric); + const metricOutputType = getMetricSpecOutputType(currentMetric); + if (singleFieldName && metricOutputType) { + typeHints[singleFieldName] = metricOutputType; + } + } + + return typeHints; } -const FILTER_FORM_FIELDS: Field[] = [ - { - name: 'type', - type: 'string', - suggestions: ['selector', 'in', 'regex', 'like', 'not'], - }, - { - name: 'dimension', - type: 'string', - defined: (df: DruidFilter) => ['selector', 'in', 'regex', 'like'].includes(df.type), - }, - { - name: 'value', - type: 'string', - defined: (df: DruidFilter) => df.type === 'selector', - }, - { - name: 'values', - type: 'string-array', - defined: (df: DruidFilter) => df.type === 'in', - }, - { - name: 'pattern', - type: 'string', - defined: (df: DruidFilter) => ['regex', 'like'].includes(df.type), - }, +export function updateSchemaWithSample( + spec: IngestionSpec, + headerAndRows: HeaderAndRows, + dimensionMode: DimensionMode, + rollup: boolean, +): IngestionSpec { + const typeHints = getTypeHintsFromSpec(spec); - { - name: 'field.type', - label: 'Sub-filter type', - type: 'string', - suggestions: ['selector', 'in', 'regex', 'like'], - defined: (df: DruidFilter) => df.type === 'not', - }, - { - name: 'field.dimension', - label: 'Sub-filter dimension', - type: 'string', - defined: (df: DruidFilter) => df.type === 'not', - }, - { - name: 'field.value', - label: 'Sub-filter value', - type: 'string', - defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'selector', - }, - { - name: 'field.values', - label: 'Sub-filter values', - type: 'string-array', - defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'in', - }, - { - name: 'field.pattern', - label: 'Sub-filter pattern', - type: 'string', - defined: (df: DruidFilter) => - df.type === 'not' && ['regex', 'like'].includes(deepGet(df, 'field.type')), - }, -]; + let newSpec = spec; -export function getFilterFormFields() { - return FILTER_FORM_FIELDS; + if (dimensionMode === 'auto-detect') { + newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', []); + } else { + newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions'); + + const dimensions = getDimensionSpecs(headerAndRows, typeHints, rollup); + if (dimensions) { + newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', dimensions); + } + } + + if (rollup) { + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'HOUR'); + + const metrics = getMetricSpecs(headerAndRows, typeHints); + if (metrics) { + newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics); + } + } else { + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'NONE'); + newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec'); + } + + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.rollup', rollup); + return newSpec; } +// ------------------------ + export function upgradeSpec(spec: any): any { if (deepGet(spec, 'spec.ioConfig.firehose')) { switch (deepGet(spec, 'spec.ioConfig.firehose.type')) { diff --git a/web-console/src/druid-models/input-format.tsx b/web-console/src/druid-models/input-format.tsx new file mode 100644 index 00000000000..15cb68258d4 --- /dev/null +++ b/web-console/src/druid-models/input-format.tsx @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; + +import { AutoForm, ExternalLink, Field } from '../components'; +import { getLink } from '../links'; +import { oneOf } from '../utils'; + +import { FlattenSpec } from './flatten-spec'; + +export interface InputFormat { + type: string; + findColumnsFromHeader?: boolean; + skipHeaderRows?: number; + columns?: string[]; + listDelimiter?: string; + pattern?: string; + function?: string; + flattenSpec?: FlattenSpec; + keepNullColumns?: boolean; +} + +export const INPUT_FORMAT_FIELDS: Field[] = [ + { + name: 'type', + label: 'Input format', + type: 'string', + suggestions: ['json', 'csv', 'tsv', 'regex', 'parquet', 'orc', 'avro_ocf'], + required: true, + info: ( + <> +

The parser used to parse the data.

+

+ For more information see{' '} + + the documentation + + . +

+ + ), + }, + { + name: 'pattern', + type: 'string', + required: true, + defined: (p: InputFormat) => p.type === 'regex', + }, + { + name: 'function', + type: 'string', + required: true, + defined: (p: InputFormat) => p.type === 'javascript', + }, + { + name: 'findColumnsFromHeader', + type: 'boolean', + required: true, + defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'), + }, + { + name: 'skipHeaderRows', + type: 'number', + defaultValue: 0, + defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'), + min: 0, + info: ( + <> + If both skipHeaderRows and hasHeaderRow options are set, skipHeaderRows is first applied. + For example, if you set skipHeaderRows to 2 and hasHeaderRow to true, Druid will skip the + first two lines and then extract column information from the third line. + + ), + }, + { + name: 'columns', + type: 'string-array', + required: true, + defined: (p: InputFormat) => + (oneOf(p.type, 'csv', 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex', + }, + { + name: 'delimiter', + type: 'string', + defaultValue: '\t', + defined: (p: InputFormat) => p.type === 'tsv', + info: <>A custom delimiter for data values., + }, + { + name: 'listDelimiter', + type: 'string', + defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv', 'regex'), + info: <>A custom delimiter for multi-value dimensions., + }, + { + name: 'binaryAsString', + type: 'boolean', + defaultValue: false, + defined: (p: InputFormat) => oneOf(p.type, 'parquet', 'orc', 'avro_ocf'), + info: ( + <> + Specifies if the binary column which is not logically marked as a string should be treated + as a UTF-8 encoded string. + + ), + }, +]; + +export function issueWithInputFormat(inputFormat: InputFormat | undefined): string | undefined { + return AutoForm.issueWithModel(inputFormat, INPUT_FORMAT_FIELDS); +} + +export function inputFormatCanFlatten(inputFormat: InputFormat): boolean { + return oneOf(inputFormat.type, 'json', 'parquet', 'orc', 'avro_ocf'); +} diff --git a/web-console/src/druid-models/input-source.tsx b/web-console/src/druid-models/input-source.tsx new file mode 100644 index 00000000000..8c4302e28b2 --- /dev/null +++ b/web-console/src/druid-models/input-source.tsx @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function nonEmptyArray(a: any) { + return Array.isArray(a) && Boolean(a.length); +} + +export interface InputSource { + type: string; + baseDir?: string; + filter?: any; + uris?: string[]; + prefixes?: string[]; + objects?: { bucket: string; path: string }[]; + fetchTimeout?: number; + + // druid + dataSource?: string; + interval?: string; + dimensions?: string[]; + metrics?: string[]; + maxInputSegmentBytesPerTask?: number; + + // inline + data?: string; + + // hdfs + paths?: string; +} + +export function issueWithInputSource(inputSource: InputSource | undefined): string | undefined { + if (!inputSource) return 'does not exist'; + if (!inputSource.type) return 'missing a type'; + switch (inputSource.type) { + case 'local': + if (!inputSource.baseDir) return `must have a 'baseDir'`; + if (!inputSource.filter) return `must have a 'filter'`; + break; + + case 'http': + if (!nonEmptyArray(inputSource.uris)) { + return 'must have at least one uri'; + } + break; + + case 'druid': + if (!inputSource.dataSource) return `must have a 'dataSource'`; + if (!inputSource.interval) return `must have an 'interval'`; + break; + + case 'inline': + if (!inputSource.data) return `must have 'data'`; + break; + + case 's3': + case 'azure': + case 'google': + if ( + !nonEmptyArray(inputSource.uris) && + !nonEmptyArray(inputSource.prefixes) && + !nonEmptyArray(inputSource.objects) + ) { + return 'must have at least one uri or prefix or object'; + } + break; + + case 'hdfs': + if (!inputSource.paths) { + return 'must have paths'; + } + break; + } + return; +} diff --git a/web-console/src/druid-models/lookup-spec.spec.ts b/web-console/src/druid-models/lookup-spec.spec.ts new file mode 100644 index 00000000000..0978f7cf97c --- /dev/null +++ b/web-console/src/druid-models/lookup-spec.spec.ts @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isLookupInvalid } from './lookup-spec'; + +describe('lookup-spec', () => { + describe('Type Map Should be disabled', () => { + it('Missing LookupName', () => { + expect(isLookupInvalid(undefined, 'v1', '__default', { type: '' })).toBe(true); + }); + + it('Empty version', () => { + expect(isLookupInvalid('lookup', '', '__default', { type: '' })).toBe(true); + }); + + it('Missing version', () => { + expect(isLookupInvalid('lookup', undefined, '__default', { type: '' })).toBe(true); + }); + + it('Empty tier', () => { + expect(isLookupInvalid('lookup', 'v1', '', { type: '' })).toBe(true); + }); + + it('Missing tier', () => { + expect(isLookupInvalid('lookup', 'v1', undefined, { type: '' })).toBe(true); + }); + + it('Missing spec', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', {})).toBe(true); + }); + + it('Type undefined', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', { type: undefined })).toBe(true); + }); + + it('Lookup of type map with no map', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', { type: 'map' })).toBe(true); + }); + + it('Lookup of type cachedNamespace with no extractionNamespace', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', { type: 'cachedNamespace' })).toBe(true); + }); + + it('Lookup of type cachedNamespace with extractionNamespace type uri, format csv, no namespaceParseSpec', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Lookup of type cachedNamespace with extractionNamespace type uri, format csv, no columns and no hasHeaderRow', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'csv', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Lookup of type cachedNamespace with extractionNamespace type uri, format tsv, no columns', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'tsv', + skipHeaderRows: 0, + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Lookup of type cachedNamespace with extractionNamespace type customJson, format tsv, no keyFieldName', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'customJson', + valueFieldName: 'value', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Lookup of type cachedNamespace with extractionNamespace type customJson, format customJson, no valueFieldName', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'customJson', + keyFieldName: 'key', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + }); + + describe('Type cachedNamespace should be disabled', () => { + it('No extractionNamespace', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', { type: 'cachedNamespace' })).toBe(true); + }); + + describe('ExtractionNamespace type URI', () => { + it('Format csv, no namespaceParseSpec', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Format csv, no columns and skipHeaderRows', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'csv', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Format tsv, no columns', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'tsv', + skipHeaderRows: 0, + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Format tsv, no keyFieldName', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'customJson', + valueFieldName: 'value', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + + it('Format customJson, no valueFieldName', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'customJson', + keyFieldName: 'key', + }, + pollPeriod: 'PT5M', + }, + }), + ).toBe(true); + }); + }); + + describe('ExtractionNamespace type JDBC', () => { + it('No namespace', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: undefined, + connectorConfig: { + createTables: true, + connectURI: 'jdbc:mysql://localhost:3306/druid', + user: 'druid', + password: 'diurd', + }, + table: 'some_lookup_table', + keyColumn: 'the_old_dim_value', + valueColumn: 'the_new_dim_value', + tsColumn: 'timestamp_column', + pollPeriod: 600000, + }, + }), + ).toBe(true); + }); + + it('No connectorConfig', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: 'some_lookup', + connectorConfig: undefined, + table: 'some_lookup_table', + keyColumn: 'the_old_dim_value', + valueColumn: 'the_new_dim_value', + tsColumn: 'timestamp_column', + pollPeriod: 600000, + }, + }), + ).toBe(true); + }); + + it('No table', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: 'some_lookup', + connectorConfig: { + createTables: true, + connectURI: 'jdbc:mysql://localhost:3306/druid', + user: 'druid', + password: 'diurd', + }, + table: undefined, + keyColumn: 'the_old_dim_value', + valueColumn: 'the_new_dim_value', + tsColumn: 'timestamp_column', + pollPeriod: 600000, + }, + }), + ).toBe(true); + }); + + it('No keyColumn', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: 'some_lookup', + connectorConfig: { + createTables: true, + connectURI: 'jdbc:mysql://localhost:3306/druid', + user: 'druid', + password: 'diurd', + }, + table: 'some_lookup_table', + keyColumn: undefined, + valueColumn: 'the_new_dim_value', + tsColumn: 'timestamp_column', + pollPeriod: 600000, + }, + }), + ).toBe(true); + }); + + it('No keyColumn', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: 'some_lookup', + connectorConfig: { + createTables: true, + connectURI: 'jdbc:mysql://localhost:3306/druid', + user: 'druid', + password: 'diurd', + }, + table: 'some_lookup_table', + keyColumn: 'the_old_dim_value', + valueColumn: undefined, + tsColumn: 'timestamp_column', + pollPeriod: 600000, + }, + }), + ).toBe(true); + }); + }); + }); + + describe('Type Map Should be enabled', () => { + it('Has type and has Map', () => { + expect(isLookupInvalid('lookup', 'v1', '__default', { type: 'map', map: { a: 'b' } })).toBe( + false, + ); + }); + }); + + describe('Type cachedNamespace Should be enabled', () => { + describe('ExtractionNamespace type URI', () => { + it('Format csv with columns', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'csv', + columns: ['key', 'value'], + }, + }, + }), + ).toBe(false); + }); + + it('Format csv with hasHeaderRow', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'csv', + hasHeaderRow: true, + }, + }, + }), + ).toBe(false); + }); + + it('Format tsv, only columns', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'tsv', + columns: ['key', 'value'], + }, + }, + }), + ).toBe(false); + }); + + it('Format tsv, keyFieldName and valueFieldName', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'uri', + uriPrefix: 's3://bucket/some/key/prefix/', + fileRegex: 'renames-[0-9]*\\.gz', + namespaceParseSpec: { + format: 'customJson', + valueFieldName: 'value', + keyFieldName: 'value', + }, + }, + }), + ).toBe(false); + }); + }); + + describe('ExtractionNamespace type JDBC', () => { + it('No namespace', () => { + expect( + isLookupInvalid('lookup', 'v1', '__default', { + type: 'cachedNamespace', + extractionNamespace: { + type: 'jdbc', + namespace: 'lookup', + connectorConfig: { + createTables: true, + connectURI: 'jdbc:mysql://localhost:3306/druid', + user: 'druid', + password: 'diurd', + }, + table: 'some_lookup_table', + keyColumn: 'the_old_dim_value', + valueColumn: 'the_new_dim_value', + }, + }), + ).toBe(false); + }); + }); + }); +}); diff --git a/web-console/src/druid-models/lookup-spec.tsx b/web-console/src/druid-models/lookup-spec.tsx new file mode 100644 index 00000000000..c9e0e5a13bd --- /dev/null +++ b/web-console/src/druid-models/lookup-spec.tsx @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { AutoForm, Field } from '../components'; +import { deepGet, deepSet, oneOf } from '../utils'; + +export interface ExtractionNamespaceSpec { + type?: string; + uri?: string; + uriPrefix?: string; + fileRegex?: string; + namespaceParseSpec?: NamespaceParseSpec; + namespace?: string; + connectorConfig?: { + createTables: boolean; + connectURI: string; + user: string; + password: string; + }; + table?: string; + keyColumn?: string; + valueColumn?: string; + filter?: any; + tsColumn?: string; + pollPeriod?: number | string; +} + +export interface NamespaceParseSpec { + format: string; + columns?: string[]; + keyColumn?: string; + valueColumn?: string; + hasHeaderRow?: boolean; + skipHeaderRows?: number; + keyFieldName?: string; + valueFieldName?: string; + delimiter?: string; + listDelimiter?: string; +} + +export interface LookupSpec { + type?: string; + map?: Record; + extractionNamespace?: ExtractionNamespaceSpec; + firstCacheTimeout?: number; + injective?: boolean; +} + +export const LOOKUP_FIELDS: Field[] = [ + { + name: 'type', + type: 'string', + suggestions: ['map', 'cachedNamespace'], + required: true, + adjustment: (model: LookupSpec) => { + if (model.type === 'map' && !model.map) { + return deepSet(model, 'map', {}); + } + if (model.type === 'cachedNamespace' && !deepGet(model, 'extractionNamespace.type')) { + return deepSet(model, 'extractionNamespace', { type: 'uri' }); + } + return model; + }, + }, + + // map lookups are simple + { + name: 'map', + type: 'json', + height: '60vh', + defined: (model: LookupSpec) => model.type === 'map', + required: true, + issueWithValue: value => { + if (!value) return 'map must be defined'; + if (typeof value !== 'object') return `map must be an object`; + for (const k in value) { + const typeValue = typeof value[k]; + if (typeValue !== 'string' && typeValue !== 'number') { + return `map key '${k}' is of the wrong type '${typeValue}'`; + } + } + return; + }, + }, + + // cachedNamespace lookups have more options + { + name: 'extractionNamespace.type', + type: 'string', + label: 'Globally cached lookup type', + placeholder: 'uri', + suggestions: ['uri', 'jdbc'], + defined: (model: LookupSpec) => model.type === 'cachedNamespace', + required: true, + }, + { + name: 'extractionNamespace.uriPrefix', + type: 'string', + label: 'URI prefix', + placeholder: 's3://bucket/some/key/prefix/', + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + !deepGet(model, 'extractionNamespace.uri'), + required: (model: LookupSpec) => + !deepGet(model, 'extractionNamespace.uriPrefix') && + !deepGet(model, 'extractionNamespace.uri'), + info: + 'A URI which specifies a directory (or other searchable resource) in which to search for files', + }, + { + name: 'extractionNamespace.uri', + type: 'string', + label: 'URI (deprecated)', + placeholder: 's3://bucket/some/key/prefix/lookups-01.gz', + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + !deepGet(model, 'extractionNamespace.uriPrefix'), + required: (model: LookupSpec) => + !deepGet(model, 'extractionNamespace.uriPrefix') && + !deepGet(model, 'extractionNamespace.uri'), + info: ( + <> +

URI for the file of interest, specified as a file, hdfs, or s3 path

+

The URI prefix option is strictly better than URI and should be used instead

+ + ), + }, + { + name: 'extractionNamespace.fileRegex', + type: 'string', + label: 'File regex', + defaultValue: '.*', + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + Boolean(deepGet(model, 'extractionNamespace.uriPrefix')), + info: 'Optional regex for matching the file name under uriPrefix.', + }, + + // namespaceParseSpec + { + name: 'extractionNamespace.namespaceParseSpec.format', + type: 'string', + label: 'Parse format', + suggestions: ['csv', 'tsv', 'simpleJson', 'customJson'], + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'uri', + required: true, + info: ( + <> +

The format of the data in the lookup files.

+

+ The simpleJson lookupParseSpec does not take any parameters. It is simply a + line delimited JSON file where the field is the key, and the field's value is the value. +

+ + ), + }, + + // CSV + TSV + { + name: 'extractionNamespace.namespaceParseSpec.skipHeaderRows', + type: 'number', + label: 'Skip header rows', + defaultValue: 0, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'), + info: `Number of header rows to be skipped. The default number of header rows to be skipped is 0.`, + }, + { + name: 'extractionNamespace.namespaceParseSpec.hasHeaderRow', + type: 'boolean', + label: 'Has header row', + defaultValue: false, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'), + info: `A flag to indicate that column information can be extracted from the input files' header row`, + }, + { + name: 'extractionNamespace.namespaceParseSpec.columns', + type: 'string-array', + label: 'Columns', + placeholder: `["key", "value"]`, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'), + required: (model: LookupSpec) => + !deepGet(model, 'extractionNamespace.namespaceParseSpec.hasHeaderRow'), + info: 'The list of columns in the csv file', + }, + { + name: 'extractionNamespace.namespaceParseSpec.keyColumn', + type: 'string', + label: 'Key column', + placeholder: '(optional - defaults to the first column)', + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'), + info: 'The name of the column containing the key', + }, + { + name: 'extractionNamespace.namespaceParseSpec.valueColumn', + type: 'string', + label: 'Value column', + placeholder: '(optional - defaults to the second column)', + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'), + info: 'The name of the column containing the value', + }, + + // TSV only + { + name: 'extractionNamespace.namespaceParseSpec.delimiter', + type: 'string', + label: 'Delimiter', + placeholder: `(optional)`, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'tsv', + }, + { + name: 'extractionNamespace.namespaceParseSpec.listDelimiter', + type: 'string', + label: 'List delimiter', + placeholder: `(optional)`, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'tsv', + }, + + // Custom JSON + { + name: 'extractionNamespace.namespaceParseSpec.keyFieldName', + type: 'string', + label: 'Key field name', + placeholder: `key`, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'customJson', + required: true, + }, + { + name: 'extractionNamespace.namespaceParseSpec.valueFieldName', + type: 'string', + label: 'Value field name', + placeholder: `value`, + defined: (model: LookupSpec) => + deepGet(model, 'extractionNamespace.type') === 'uri' && + deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'customJson', + required: true, + }, + { + name: 'extractionNamespace.pollPeriod', + type: 'string', + label: 'Poll period', + defaultValue: '0', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'uri', + info: `Period between polling for updates`, + }, + + // JDBC stuff + { + name: 'extractionNamespace.namespace', + type: 'string', + label: 'Namespace', + placeholder: 'some_lookup', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + required: true, + info: ( + <> +

The namespace value in the SQL query:

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE + filter +

+ + ), + }, + { + name: 'extractionNamespace.connectorConfig.connectURI', + type: 'string', + label: 'Connect URI', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + required: true, + info: 'Defines the connectURI value on the The connector config to used', + }, + { + name: 'extractionNamespace.connectorConfig.user', + type: 'string', + label: 'User', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + info: 'Defines the user to be used by the connector config', + }, + { + name: 'extractionNamespace.connectorConfig.password', + type: 'string', + label: 'Password', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + info: 'Defines the password to be used by the connector config', + }, + { + name: 'extractionNamespace.connectorConfig.createTables', + type: 'boolean', + label: 'Create tables', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + info: 'Should tables be created', + }, + { + name: 'extractionNamespace.table', + type: 'string', + label: 'Table', + placeholder: 'some_lookup_table', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + required: true, + info: ( + <> +

+ The table which contains the key value pairs. This will become the table value in the SQL + query: +

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE + filter +

+ + ), + }, + { + name: 'extractionNamespace.keyColumn', + type: 'string', + label: 'Key column', + placeholder: 'my_key_value', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + required: true, + info: ( + <> +

+ The column in the table which contains the keys. This will become the keyColumn value in + the SQL query: +

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE + filter +

+ + ), + }, + { + name: 'extractionNamespace.valueColumn', + type: 'string', + label: 'Value column', + placeholder: 'my_column_value', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + required: true, + info: ( + <> +

+ The column in table which contains the values. This will become the valueColumn value in + the SQL query: +

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE + filter +

+ + ), + }, + { + name: 'extractionNamespace.filter', + type: 'string', + label: 'Filter', + placeholder: '(optional)', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + info: ( + <> +

+ The filter to be used when selecting lookups, this is used to create a where clause on + lookup population. This will become the expression filter in the SQL query: +

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE{' '} + filter +

+ + ), + }, + { + name: 'extractionNamespace.tsColumn', + type: 'string', + label: 'Timestamp column', + placeholder: '(optional)', + defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc', + info: ( + <> +

+ The column in table which contains when the key was updated. This will become the Value in + the SQL query: +

+

+ SELECT keyColumn, valueColumn, tsColumn? FROM namespace.table WHERE + filter +

+ + ), + }, + + // Extra cachedNamespace things + { + name: 'firstCacheTimeout', + type: 'number', + label: 'First cache timeout', + defaultValue: 0, + defined: (model: LookupSpec) => model.type === 'cachedNamespace', + info: `How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait`, + }, + { + name: 'injective', + type: 'boolean', + defaultValue: false, + defined: (model: LookupSpec) => model.type === 'cachedNamespace', + info: `If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to true`, + }, +]; + +export function isLookupInvalid( + lookupName: string | undefined, + lookupVersion: string | undefined, + lookupTier: string | undefined, + lookupSpec: LookupSpec | undefined, +) { + return ( + !lookupName || + !lookupVersion || + !lookupTier || + Boolean(AutoForm.issueWithModel(lookupSpec, LOOKUP_FIELDS)) + ); +} diff --git a/web-console/src/druid-models/metric-spec.spec.ts b/web-console/src/druid-models/metric-spec.spec.ts new file mode 100644 index 00000000000..25b3f1572e9 --- /dev/null +++ b/web-console/src/druid-models/metric-spec.spec.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getMetricSpecs } from './metric-spec'; + +describe('metric-spec', () => { + it('getMetricSecs', () => { + expect(getMetricSpecs({ header: ['header'], rows: [] }, {})).toMatchInlineSnapshot(` + Array [ + Object { + "name": "count", + "type": "count", + }, + ] + `); + }); +}); diff --git a/web-console/src/druid-models/metric-spec.tsx b/web-console/src/druid-models/metric-spec.tsx new file mode 100644 index 00000000000..fd1282fe770 --- /dev/null +++ b/web-console/src/druid-models/metric-spec.tsx @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { ExternalLink, Field } from '../components'; +import { getLink } from '../links'; +import { filterMap, oneOf } from '../utils'; +import { HeaderAndRows } from '../utils/sampler'; + +import { getColumnTypeFromHeaderAndRows } from './ingestion-spec'; + +export interface MetricSpec { + type: string; + name?: string; + fieldName?: string; + maxStringBytes?: number; + filterNullValues?: boolean; + fieldNames?: string[]; + fnAggregate?: string; + fnCombine?: string; + fnReset?: string; + fields?: string[]; + byRow?: boolean; + round?: boolean; + isInputHyperUnique?: boolean; + filter?: any; + aggregator?: MetricSpec; +} + +export const METRIC_SPEC_FIELDS: Field[] = [ + { + name: 'name', + type: 'string', + info: <>The metric name as it will appear in Druid., + }, + { + name: 'type', + type: 'string', + suggestions: [ + 'count', + { + group: 'sum', + suggestions: ['longSum', 'doubleSum', 'floatSum'], + }, + { + group: 'min', + suggestions: ['longMin', 'doubleMin', 'floatMin'], + }, + { + group: 'max', + suggestions: ['longMax', 'doubleMax', 'floatMax'], + }, + { + group: 'first', + suggestions: ['longFirst', 'doubleFirst', 'floatFirst'], + }, + { + group: 'last', + suggestions: ['longLast', 'doubleLast', 'floatLast'], + }, + 'thetaSketch', + { + group: 'HLLSketch', + suggestions: ['HLLSketchBuild', 'HLLSketchMerge'], + }, + 'quantilesDoublesSketch', + 'momentSketch', + 'fixedBucketsHistogram', + 'hyperUnique', + 'filtered', + ], + info: <>The aggregation function to apply., + }, + { + name: 'fieldName', + type: 'string', + defined: m => m.type !== 'filtered', + info: <>The column name for the aggregator to operate on., + }, + { + name: 'maxStringBytes', + type: 'number', + defaultValue: 1024, + defined: m => { + return oneOf(m.type, 'stringFirst', 'stringLast'); + }, + }, + { + name: 'filterNullValues', + type: 'boolean', + defaultValue: false, + defined: m => { + return oneOf(m.type, 'stringFirst', 'stringLast'); + }, + }, + // filtered + { + name: 'filter', + type: 'json', + defined: m => m.type === 'filtered', + }, + { + name: 'aggregator', + type: 'json', + defined: m => m.type === 'filtered', + }, + // thetaSketch + { + name: 'size', + type: 'number', + defined: m => m.type === 'thetaSketch', + defaultValue: 16384, + info: ( + <> +

+ Must be a power of 2. Internally, size refers to the maximum number of entries sketch + object will retain. Higher size means higher accuracy but more space to store sketches. + Note that after you index with a particular size, druid will persist sketch in segments + and you will use size greater or equal to that at query time. +

+

+ See the{' '} + + DataSketches site + {' '} + for details. +

+

In general, We recommend just sticking to default size.

+ + ), + }, + { + name: 'isInputThetaSketch', + type: 'boolean', + defined: m => m.type === 'thetaSketch', + defaultValue: false, + info: ( + <> + This should only be used at indexing time if your input data contains theta sketch objects. + This would be the case if you use datasketches library outside of Druid, say with Pig/Hive, + to produce the data that you are ingesting into Druid + + ), + }, + // HLLSketchBuild & HLLSketchMerge + { + name: 'lgK', + type: 'number', + defined: m => oneOf(m.type, 'HLLSketchBuild', 'HLLSketchMerge'), + defaultValue: 12, + info: ( + <> +

+ log2 of K that is the number of buckets in the sketch, parameter that controls the size + and the accuracy. +

+

Must be between 4 to 21 inclusively.

+ + ), + }, + { + name: 'tgtHllType', + type: 'string', + defined: m => oneOf(m.type, 'HLLSketchBuild', 'HLLSketchMerge'), + defaultValue: 'HLL_4', + suggestions: ['HLL_4', 'HLL_6', 'HLL_8'], + info: ( + <> + The type of the target HLL sketch. Must be HLL_4, HLL_6, or{' '} + HLL_8. + + ), + }, + // quantilesDoublesSketch + { + name: 'k', + type: 'number', + defined: m => m.type === 'quantilesDoublesSketch', + defaultValue: 128, + info: ( + <> +

+ Parameter that determines the accuracy and size of the sketch. Higher k means higher + accuracy but more space to store sketches. +

+

+ Must be a power of 2 from 2 to 32768. See the{' '} + + Quantiles Accuracy + {' '} + for details. +

+ + ), + }, + // momentSketch + { + name: 'k', + type: 'number', + defined: m => m.type === 'momentSketch', + required: true, + info: ( + <> + Parameter that determines the accuracy and size of the sketch. Higher k means higher + accuracy but more space to store sketches. Usable range is generally [3,15] + + ), + }, + { + name: 'compress', + type: 'boolean', + defined: m => m.type === 'momentSketch', + defaultValue: true, + info: ( + <> + Flag for whether the aggregator compresses numeric values using arcsinh. Can improve + robustness to skewed and long-tailed distributions, but reduces accuracy slightly on more + uniform distributions. + + ), + }, + // fixedBucketsHistogram + { + name: 'lowerLimit', + type: 'number', + defined: m => m.type === 'fixedBucketsHistogram', + required: true, + info: <>Lower limit of the histogram., + }, + { + name: 'upperLimit', + type: 'number', + defined: m => m.type === 'fixedBucketsHistogram', + required: true, + info: <>Upper limit of the histogram., + }, + { + name: 'numBuckets', + type: 'number', + defined: m => m.type === 'fixedBucketsHistogram', + defaultValue: 10, + required: true, + info: ( + <> + Number of buckets for the histogram. The range [lowerLimit, upperLimit] will be + divided into numBuckets intervals of equal size. + + ), + }, + { + name: 'outlierHandlingMode', + type: 'string', + defined: m => m.type === 'fixedBucketsHistogram', + required: true, + suggestions: ['ignore', 'overflow', 'clip'], + info: ( + <> +

+ Specifies how values outside of [lowerLimit, upperLimit] will be handled. +

+

+ Supported modes are ignore, overflow, and clip. See + + outlier handling modes + {' '} + for more details. +

+ + ), + }, + // hyperUnique + { + name: 'isInputHyperUnique', + type: 'boolean', + defined: m => m.type === 'hyperUnique', + defaultValue: false, + info: ( + <> + This can be set to true to index precomputed HLL (Base64 encoded output from druid-hll is + expected). + + ), + }, +]; + +export function getMetricSpecName(metricSpec: MetricSpec): string { + return ( + metricSpec.name || (metricSpec.aggregator ? getMetricSpecName(metricSpec.aggregator) : '?') + ); +} + +export function getMetricSpecSingleFieldName(metricSpec: MetricSpec): string | undefined { + return ( + metricSpec.fieldName || + (metricSpec.aggregator ? getMetricSpecSingleFieldName(metricSpec.aggregator) : undefined) + ); +} + +export function getMetricSpecOutputType(metricSpec: MetricSpec): string | undefined { + if (metricSpec.aggregator) return getMetricSpecOutputType(metricSpec.aggregator); + const m = String(metricSpec.type).match(/^(long|float|double)/); + if (!m) return; + return m[1]; +} + +export function getMetricSpecs( + headerAndRows: HeaderAndRows, + typeHints: Record, +): MetricSpec[] { + return [{ name: 'count', type: 'count' }].concat( + filterMap(headerAndRows.header, h => { + if (h === '__time') return; + const type = typeHints[h] || getColumnTypeFromHeaderAndRows(headerAndRows, h); + switch (type) { + case 'double': + return { name: `sum_${h}`, type: 'doubleSum', fieldName: h }; + case 'float': + return { name: `sum_${h}`, type: 'floatSum', fieldName: h }; + case 'long': + return { name: `sum_${h}`, type: 'longSum', fieldName: h }; + default: + return; + } + }), + ); +} diff --git a/web-console/src/utils/druid-time.spec.ts b/web-console/src/druid-models/time.spec.ts similarity index 95% rename from web-console/src/utils/druid-time.spec.ts rename to web-console/src/druid-models/time.spec.ts index 6ebb4bceeb0..5670640f707 100644 --- a/web-console/src/utils/druid-time.spec.ts +++ b/web-console/src/druid-models/time.spec.ts @@ -16,7 +16,7 @@ * limitations under the License. */ -import { timeFormatMatches } from './druid-time'; +import { timeFormatMatches } from './time'; describe('timeFormatMatches', () => { it('works for auto', () => { diff --git a/web-console/src/utils/druid-time.ts b/web-console/src/druid-models/time.ts similarity index 98% rename from web-console/src/utils/druid-time.ts rename to web-console/src/druid-models/time.ts index 3dc6fe93554..c20d2cbbc8e 100644 --- a/web-console/src/utils/druid-time.ts +++ b/web-console/src/druid-models/time.ts @@ -16,7 +16,7 @@ * limitations under the License. */ -import { jodaFormatToRegExp } from './joda-to-regexp'; +import { jodaFormatToRegExp } from '../utils/joda-to-regexp'; export const NUMERIC_TIME_FORMATS: string[] = ['posix', 'millis', 'micro', 'nano']; export const BASIC_TIME_FORMATS: string[] = ['auto', 'iso'].concat(NUMERIC_TIME_FORMATS); diff --git a/web-console/src/druid-models/timestamp-spec.tsx b/web-console/src/druid-models/timestamp-spec.tsx new file mode 100644 index 00000000000..8e17d20dbbc --- /dev/null +++ b/web-console/src/druid-models/timestamp-spec.tsx @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; + +import { ExternalLink, Field } from '../components'; +import { deepGet, EMPTY_ARRAY, EMPTY_OBJECT } from '../utils'; + +import { IngestionSpec } from './ingestion-spec'; +import { + BASIC_TIME_FORMATS, + DATE_ONLY_TIME_FORMATS, + DATETIME_TIME_FORMATS, + OTHER_TIME_FORMATS, +} from './time'; +import { Transform } from './transform-spec'; + +const NO_SUCH_COLUMN = '!!!_no_such_column_!!!'; + +export const PLACEHOLDER_TIMESTAMP_SPEC: TimestampSpec = { + column: NO_SUCH_COLUMN, + missingValue: '1970-01-01T00:00:00Z', +}; + +export const CONSTANT_TIMESTAMP_SPEC: TimestampSpec = { + column: NO_SUCH_COLUMN, + missingValue: '2010-01-01T00:00:00Z', +}; + +export type TimestampSchema = 'none' | 'column' | 'expression'; + +export function getTimestampSchema(spec: IngestionSpec): TimestampSchema { + const transforms: Transform[] = + deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY; + + const timeTransform = transforms.find(transform => transform.name === '__time'); + if (timeTransform) return 'expression'; + + const timestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec') || EMPTY_OBJECT; + return timestampSpec.column === NO_SUCH_COLUMN ? 'none' : 'column'; +} + +export interface TimestampSpec { + column?: string; + format?: string; + missingValue?: string; +} + +export function getTimestampSpecColumnFromSpec(spec: IngestionSpec): string { + // For the default https://github.com/apache/druid/blob/master/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java#L44 + return deepGet(spec, 'spec.dataSchema.timestampSpec.column') || 'timestamp'; +} + +export function getTimestampSpecConstantFromSpec(spec: IngestionSpec): string | undefined { + return deepGet(spec, 'spec.dataSchema.timestampSpec.missingValue'); +} + +export function getTimestampSpecExpressionFromSpec(spec: IngestionSpec): string | undefined { + const transforms: Transform[] = + deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY; + + const timeTransform = transforms.find(transform => transform.name === '__time'); + if (!timeTransform) return; + return timeTransform.expression; +} + +export function getTimestampDetailFromSpec(spec: IngestionSpec): string { + const timestampSchema = getTimestampSchema(spec); + switch (timestampSchema) { + case 'none': + return `Constant: ${getTimestampSpecConstantFromSpec(spec)}`; + + case 'column': + return `Column: ${getTimestampSpecColumnFromSpec(spec)}`; + + case 'expression': + return `Expression: ${getTimestampSpecExpressionFromSpec(spec)}`; + } + + return '-'; +} + +export const TIMESTAMP_SPEC_FIELDS: Field[] = [ + { + name: 'column', + type: 'string', + defaultValue: 'timestamp', + required: true, + }, + { + name: 'format', + type: 'string', + defaultValue: 'auto', + suggestions: [ + ...BASIC_TIME_FORMATS, + { + group: 'Date and time formats', + suggestions: DATETIME_TIME_FORMATS, + }, + { + group: 'Date only formats', + suggestions: DATE_ONLY_TIME_FORMATS, + }, + { + group: 'Other time formats', + suggestions: OTHER_TIME_FORMATS, + }, + ], + info: ( +

+ Please specify your timestamp format by using the suggestions menu or typing in a{' '} + + format string + + . +

+ ), + }, + { + name: 'missingValue', + type: 'string', + placeholder: '(optional)', + info:

This value will be used if the specified column can not be found.

, + }, +]; + +export const CONSTANT_TIMESTAMP_SPEC_FIELDS: Field[] = [ + { + name: 'missingValue', + label: 'Placeholder value', + type: 'string', + info:

The placeholder value that will be used as the timestamp.

, + }, +]; + +export function issueWithTimestampSpec( + timestampSpec: TimestampSpec | undefined, +): string | undefined { + if (!timestampSpec) return 'no spec'; + if (!timestampSpec.column && !timestampSpec.missingValue) return 'timestamp spec is blank'; + return; +} diff --git a/web-console/src/druid-models/transform-spec.tsx b/web-console/src/druid-models/transform-spec.tsx new file mode 100644 index 00000000000..81e9b334e3a --- /dev/null +++ b/web-console/src/druid-models/transform-spec.tsx @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { ExternalLink, Field } from '../components'; +import { getLink } from '../links'; + +export interface TransformSpec { + transforms?: Transform[]; + filter?: Record; +} + +export interface Transform { + type: string; + name: string; + expression: string; +} + +export const TRANSFORM_FIELDS: Field[] = [ + { + name: 'name', + type: 'string', + placeholder: 'output_name', + required: true, + }, + { + name: 'type', + type: 'string', + suggestions: ['expression'], + required: true, + }, + { + name: 'expression', + type: 'string', + placeholder: '"foo" + "bar"', + required: true, + info: ( + <> + A valid Druid{' '} + expression. + + ), + }, +]; + +export function getTimestampExpressionFields(transforms: Transform[]): Field[] { + const timeTransformIndex = transforms.findIndex(transform => transform.name === '__time'); + if (timeTransformIndex < 0) return []; + + return [ + { + name: `${timeTransformIndex}.expression`, + label: 'Expression', + type: 'string', + placeholder: `timestamp_parse(concat("date", ' ', "time"))`, + required: true, + suggestions: [ + `timestamp_parse(concat("date", ' ', "time"))`, + `timestamp_parse(concat("date", ' ', "time"), 'M/d/yyyy H:mm:ss')`, + `timestamp_parse(concat("year", '-', "month", '-', "day"))`, + ], + info: ( + <> + A valid Druid{' '} + expression{' '} + that should output a millis timestamp. You most likely want to use the{' '} + timestamp_parse function at the outer level. + + ), + }, + ]; +} + +export function addTimestampTransform(transforms: Transform[]): Transform[] { + return [ + { + name: '__time', + type: 'expression', + expression: '', + }, + ].concat(transforms); +} + +export function removeTimestampTransform(transforms: Transform[]): Transform[] | undefined { + const newTransforms = transforms.filter(transform => transform.name !== '__time'); + return newTransforms.length ? newTransforms : undefined; +} diff --git a/web-console/src/entry.scss b/web-console/src/entry.scss index 38a0d1914eb..426480ea24b 100644 --- a/web-console/src/entry.scss +++ b/web-console/src/entry.scss @@ -16,13 +16,13 @@ * limitations under the License. */ -@import '../node_modules/normalize.css/normalize'; +@import '~normalize.css/normalize'; @import '~fontsource-open-sans/index.css'; @import './blueprint-overrides'; @import '~@blueprintjs/core/src/blueprint'; @import '~@blueprintjs/datetime/src/blueprint-datetime'; +@import '~react-splitter-layout/lib/index'; @import '../lib/react-table'; -@import '../node_modules/react-splitter-layout/lib/index.css'; html, body { @@ -45,6 +45,10 @@ body { outline: none !important; } } + + .ace-solarized-dark { + background-color: rgba($dark-gray1, 0.5); + } } .app-container { diff --git a/web-console/src/links.ts b/web-console/src/links.ts index b4b3b238a18..488b238e12b 100644 --- a/web-console/src/links.ts +++ b/web-console/src/links.ts @@ -19,7 +19,7 @@ import hasOwnProp from 'has-own-prop'; // This is set to the latest available version and should be updated to the next version before release -const DRUID_DOCS_VERSION = '0.19.0'; +const DRUID_DOCS_VERSION = '0.20.0'; function fillVersion(str: string): string { return str.replace(/\{\{VERSION}}/g, DRUID_DOCS_VERSION); diff --git a/web-console/src/utils/druid-query.spec.ts b/web-console/src/utils/druid-query.spec.ts index 140f146b521..55fd336920d 100644 --- a/web-console/src/utils/druid-query.spec.ts +++ b/web-console/src/utils/druid-query.spec.ts @@ -18,7 +18,7 @@ import { sane } from 'druid-query-toolkit/build/test-utils'; -import { DruidError } from './druid-query'; +import { DruidError, getDruidErrorMessage, parseHtmlError, parseQueryPlan } from './druid-query'; describe('DruidQuery', () => { describe('DruidError.parsePosition', () => { @@ -128,4 +128,18 @@ describe('DruidQuery', () => { expect(suggestion).toBeUndefined(); }); }); + + describe('misc', () => { + it('parseHtmlError', () => { + expect(parseHtmlError('
')).toMatchInlineSnapshot(`undefined`); + }); + + it('parseHtmlError', () => { + expect(getDruidErrorMessage({})).toMatchInlineSnapshot(`undefined`); + }); + + it('parseQueryPlan', () => { + expect(parseQueryPlan('start')).toMatchInlineSnapshot(`"start"`); + }); + }); }); diff --git a/web-console/src/utils/druid-type.ts b/web-console/src/utils/druid-type.ts deleted file mode 100644 index 821dddc9b55..00000000000 --- a/web-console/src/utils/druid-type.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { filterMap } from './general'; -import { DimensionMode, DimensionSpec, IngestionSpec, MetricSpec } from './ingestion-spec'; -import { deepDelete, deepSet } from './object-change'; -import { HeaderAndRows } from './sampler'; - -export function guessTypeFromSample(sample: any[]): string { - const definedValues = sample.filter(v => v != null); - if ( - definedValues.length && - definedValues.every(v => !isNaN(v) && (typeof v === 'number' || typeof v === 'string')) - ) { - if (definedValues.every(v => v % 1 === 0)) { - return 'long'; - } else { - return 'double'; - } - } else { - return 'string'; - } -} - -export function getColumnTypeFromHeaderAndRows( - headerAndRows: HeaderAndRows, - column: string, -): string { - return guessTypeFromSample( - filterMap(headerAndRows.rows, (r: any) => (r.parsed ? r.parsed[column] : undefined)), - ); -} - -export function getDimensionSpecs( - headerAndRows: HeaderAndRows, - hasRollup: boolean, -): (string | DimensionSpec)[] { - return filterMap(headerAndRows.header, h => { - if (h === '__time') return; - const guessedType = getColumnTypeFromHeaderAndRows(headerAndRows, h); - if (guessedType === 'string') return h; - if (hasRollup) return; - return { - type: guessedType, - name: h, - }; - }); -} - -export function getMetricSpecs(headerAndRows: HeaderAndRows): MetricSpec[] { - return [{ name: 'count', type: 'count' }].concat( - filterMap(headerAndRows.header, h => { - if (h === '__time') return; - const guessedType = getColumnTypeFromHeaderAndRows(headerAndRows, h); - switch (guessedType) { - case 'double': - return { name: `sum_${h}`, type: 'doubleSum', fieldName: h }; - case 'long': - return { name: `sum_${h}`, type: 'longSum', fieldName: h }; - default: - return; - } - }), - ); -} - -export function updateSchemaWithSample( - spec: IngestionSpec, - headerAndRows: HeaderAndRows, - dimensionMode: DimensionMode, - rollup: boolean, -): IngestionSpec { - let newSpec = spec; - - if (dimensionMode === 'auto-detect') { - newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', []); - } else { - newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions'); - - const dimensions = getDimensionSpecs(headerAndRows, rollup); - if (dimensions) { - newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', dimensions); - } - } - - if (rollup) { - newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'HOUR'); - - const metrics = getMetricSpecs(headerAndRows); - if (metrics) { - newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics); - } - } else { - newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'NONE'); - newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec'); - } - - newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.rollup', rollup); - return newSpec; -} diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts index a9501034819..e614b870943 100644 --- a/web-console/src/utils/general.spec.ts +++ b/web-console/src/utils/general.spec.ts @@ -55,20 +55,26 @@ describe('general', () => { }); describe('sqlQueryCustomTableFilter', () => { - it('works', () => { + it('works with contains', () => { expect( - sqlQueryCustomTableFilter({ - id: 'datasource', - value: `hello`, - }), - ).toMatchInlineSnapshot(`"LOWER(\\"datasource\\") LIKE LOWER('%hello%')"`); + String( + sqlQueryCustomTableFilter({ + id: 'datasource', + value: `Hello`, + }), + ), + ).toEqual(`LOWER("datasource") LIKE '%hello%'`); + }); + it('works with exact', () => { expect( - sqlQueryCustomTableFilter({ - id: 'datasource', - value: `"hello"`, - }), - ).toMatchInlineSnapshot(`"\\"datasource\\" = 'hello'"`); + String( + sqlQueryCustomTableFilter({ + id: 'datasource', + value: `"hello"`, + }), + ), + ).toEqual(`"datasource" = 'hello'`); }); }); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 2fc5762e34c..ca2a111199e 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -19,6 +19,7 @@ import { Button, HTMLSelect, InputGroup, Intent } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import copy from 'copy-to-clipboard'; +import { SqlExpression, SqlFunction, SqlLiteral, SqlRef } from 'druid-query-toolkit'; import FileSaver from 'file-saver'; import hasOwnProp from 'has-own-prop'; import numeral from 'numeral'; @@ -27,6 +28,10 @@ import { Filter, FilterRender } from 'react-table'; import { AppToaster } from '../singletons/toaster'; +// These constants are used to make sure that they are not constantly recreated thrashing the pure components +export const EMPTY_OBJECT: any = {}; +export const EMPTY_ARRAY: any[] = []; + export function wait(ms: number): Promise { return new Promise(resolve => { setTimeout(resolve, ms); @@ -117,14 +122,15 @@ export function booleanCustomTableFilter(filter: Filter, value: any): boolean { return haystack.includes(needle); } -export function sqlQueryCustomTableFilter(filter: Filter): string { - const columnName = JSON.stringify(filter.id); +export function sqlQueryCustomTableFilter(filter: Filter): SqlExpression { const needleAndMode: NeedleAndMode = getNeedleAndMode(filter.value); const needle = needleAndMode.needle; if (needleAndMode.mode === 'exact') { - return `${columnName} = '${needle}'`; + return SqlRef.columnWithQuotes(filter.id).equal(SqlLiteral.create(needle)); } else { - return `LOWER(${columnName}) LIKE LOWER('%${needle}%')`; + return SqlFunction.simple('LOWER', [SqlRef.columnWithQuotes(filter.id)]).like( + SqlLiteral.create(`%${needle.toLowerCase()}%`), + ); } } @@ -135,6 +141,10 @@ export function caseInsensitiveContains(testString: string, searchString: string return testString.toLowerCase().includes(searchString.toLowerCase()); } +export function oneOf(thing: T, ...options: T[]): boolean { + return options.includes(thing); +} + // ---------------------------- export function countBy( diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx index 2bcf661c7f9..d27831794d1 100644 --- a/web-console/src/utils/index.tsx +++ b/web-console/src/utils/index.tsx @@ -24,4 +24,5 @@ export * from './query-manager'; export * from './query-cursor'; export * from './local-storage-keys'; export * from './column-metadata'; -export * from './compaction'; +export * from './object-change'; +export * from './capabilities'; diff --git a/web-console/src/utils/object-change.ts b/web-console/src/utils/object-change.ts index 83b7cceaa2b..7ff7d5e2fc8 100644 --- a/web-console/src/utils/object-change.ts +++ b/web-console/src/utils/object-change.ts @@ -83,6 +83,17 @@ export function deepSet>(value: T, path: string, x return valueCopy; } +export function deepSetMulti>( + value: T, + changes: Record, +): T { + let newValue = value; + for (const k in changes) { + newValue = deepSet(newValue, k, changes[k]); + } + return newValue; +} + export function deepDelete>(value: T, path: string): T { const valueCopy = shallowCopy(value); const parts = parsePath(path); diff --git a/web-console/src/utils/query-manager.tsx b/web-console/src/utils/query-manager.tsx index 411d0541486..906c7b6bd27 100644 --- a/web-console/src/utils/query-manager.tsx +++ b/web-console/src/utils/query-manager.tsx @@ -165,5 +165,8 @@ export class QueryManager { public terminate(): void { this.terminated = true; + if (this.currentRunCancelFn) { + this.currentRunCancelFn(); + } } } diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index c56f57294e5..48db5f6c19b 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -18,24 +18,31 @@ import axios from 'axios'; -import { getDruidErrorMessage, queryDruidRune } from './druid-query'; -import { alphanumericCompare, filterMap, sortWithPrefixSuffix } from './general'; import { DimensionsSpec, - getDummyTimestampSpec, getSpecType, + getTimestampSchema, IngestionSpec, IngestionType, InputFormat, IoConfig, - isColumnTimestampSpec, isDruidSource, MetricSpec, + PLACEHOLDER_TIMESTAMP_SPEC, TimestampSpec, Transform, TransformSpec, upgradeSpec, -} from './ingestion-spec'; +} from '../druid-models'; + +import { getDruidErrorMessage, queryDruidRune } from './druid-query'; +import { + alphanumericCompare, + EMPTY_ARRAY, + filterMap, + oneOf, + sortWithPrefixSuffix, +} from './general'; import { deepGet, deepSet } from './object-change'; const SAMPLER_URL = `/druid/indexer/v1/sampler`; @@ -231,7 +238,8 @@ function cleanupQueryGranularity(queryGranularity: any): any { if (typeof queryGranularityType !== 'string') return queryGranularity; queryGranularityType = queryGranularityType.toUpperCase(); - const knownGranularity = [ + const knownGranularity = oneOf( + queryGranularityType, 'NONE', 'SECOND', 'MINUTE', @@ -240,7 +248,7 @@ function cleanupQueryGranularity(queryGranularity: any): any { 'WEEK', 'MONTH', 'YEAR', - ].includes(queryGranularityType); + ); return knownGranularity ? queryGranularityType : queryGranularity; } @@ -272,7 +280,7 @@ export async function sampleForConnect( ioConfig, dataSchema: { dataSource: 'sample', - timestampSpec: getDummyTimestampSpec(), + timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC, dimensionsSpec: {}, }, } as any, @@ -326,7 +334,7 @@ export async function sampleForParser( ioConfig, dataSchema: { dataSource: 'sample', - timestampSpec: getDummyTimestampSpec(), + timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC, dimensionsSpec: {}, }, }, @@ -342,7 +350,7 @@ export async function sampleForTimestamp( ): Promise { const samplerType = getSpecType(spec); const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec'); - const columnTimestampSpec = isColumnTimestampSpec(timestampSpec); + const timestampSchema = getTimestampSchema(spec); // First do a query with a static timestamp spec const sampleSpecColumns: SampleSpec = { @@ -352,7 +360,7 @@ export async function sampleForTimestamp( dataSchema: { dataSource: 'sample', dimensionsSpec: {}, - timestampSpec: columnTimestampSpec ? getDummyTimestampSpec() : timestampSpec, + timestampSpec: timestampSchema === 'column' ? PLACEHOLDER_TIMESTAMP_SPEC : timestampSpec, }, }, samplerConfig: BASE_SAMPLER_CONFIG, @@ -364,7 +372,10 @@ export async function sampleForTimestamp( ); // If we are not parsing a column then there is nothing left to do - if (!columnTimestampSpec) return sampleColumns; + if (timestampSchema === 'none') return sampleColumns; + + const transforms: Transform[] = + deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY; // If we are trying to parts a column then get a bit fancy: // Query the same sample again (same cache key) @@ -376,6 +387,9 @@ export async function sampleForTimestamp( dataSource: 'sample', dimensionsSpec: {}, timestampSpec, + transformSpec: { + transforms: transforms.filter(transform => transform.name === '__time'), + }, }, }, samplerConfig: BASE_SAMPLER_CONFIG, diff --git a/web-console/src/utils/utils.spec.ts b/web-console/src/utils/utils.spec.ts index cc85278666c..b2ddb024577 100644 --- a/web-console/src/utils/utils.spec.ts +++ b/web-console/src/utils/utils.spec.ts @@ -16,18 +16,11 @@ * limitations under the License. */ -import { getDruidErrorMessage, parseHtmlError, parseQueryPlan } from './druid-query'; -import { - getColumnTypeFromHeaderAndRows, - getDimensionSpecs, - getMetricSpecs, - guessTypeFromSample, - updateSchemaWithSample, -} from './druid-type'; -import { IngestionSpec } from './ingestion-spec'; +import { IngestionSpec } from '../druid-models'; + import { applyCache, headerFromSampleResponse } from './sampler'; -describe('test-utils', () => { +describe('utils', () => { const ingestionSpec: IngestionSpec = { type: 'index_parallel', spec: { @@ -123,161 +116,4 @@ describe('test-utils', () => { } `); }); - - // it('spec-utils sampleForParser', async () => { - // expect(await sampleForParser(ingestionSpec, 'start', 'abc123')).toMatchInlineSnapshot( - // `Promise {}`, - // ); - // }); - // - // it('spec-utils SampleSpec', async () => { - // expect(await sampleForConnect(ingestionSpec, 'start')).toMatchInlineSnapshot(`Promise {}`); - // }); - // - // it('spec-utils sampleForTimestamp', async () => { - // expect(await sampleForTimestamp(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot(); - // }); - // - // it('spec-utils sampleForTransform', async () => { - // expect(await sampleForTransform(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot(); - // }); - // - // it('spec-utils sampleForFilter', async () => { - // expect(await sampleForFilter(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot(); - // }); - // - // it('spec-utils sampleForSchema', async () => { - // expect(await sampleForSchema(ingestionSpec, 'start', cacheRows)).toMatchInlineSnapshot(); - // }); - // - // it('spec-utils sampleForExampleManifests', async () => { - // expect(await sampleForExampleManifests('some url')).toMatchInlineSnapshot(); - // }); -}); - -describe('druid-type.ts', () => { - const ingestionSpec: IngestionSpec = { - type: 'index_parallel', - spec: { - ioConfig: { - type: 'index_parallel', - inputSource: { - type: 'http', - uris: ['https://static.imply.io/data/wikipedia.json.gz'], - }, - inputFormat: { - type: 'json', - }, - }, - tuningConfig: { - type: 'index_parallel', - }, - dataSchema: { - dataSource: 'wikipedia', - granularitySpec: { - type: 'uniform', - segmentGranularity: 'DAY', - queryGranularity: 'HOUR', - }, - timestampSpec: { - column: 'timestamp', - format: 'iso', - }, - dimensionsSpec: {}, - }, - }, - }; - - it('spec-utils guessTypeFromSample', () => { - expect(guessTypeFromSample([])).toMatchInlineSnapshot(`"string"`); - }); - - it('spec-utils getColumnTypeFromHeaderAndRows', () => { - expect( - getColumnTypeFromHeaderAndRows({ header: ['header'], rows: [] }, 'header'), - ).toMatchInlineSnapshot(`"string"`); - }); - - it('spec-utils getDimensionSpecs', () => { - expect(getDimensionSpecs({ header: ['header'], rows: [] }, true)).toMatchInlineSnapshot(` - Array [ - "header", - ] - `); - }); - - it('spec-utils getMetricSecs', () => { - expect(getMetricSpecs({ header: ['header'], rows: [] })).toMatchInlineSnapshot(` - Array [ - Object { - "name": "count", - "type": "count", - }, - ] - `); - }); - - it('spec-utils updateSchemaWithSample', () => { - expect( - updateSchemaWithSample(ingestionSpec, { header: ['header'], rows: [] }, 'specific', true), - ).toMatchInlineSnapshot(` - Object { - "spec": Object { - "dataSchema": Object { - "dataSource": "wikipedia", - "dimensionsSpec": Object { - "dimensions": Array [ - "header", - ], - }, - "granularitySpec": Object { - "queryGranularity": "HOUR", - "rollup": true, - "segmentGranularity": "DAY", - "type": "uniform", - }, - "metricsSpec": Array [ - Object { - "name": "count", - "type": "count", - }, - ], - "timestampSpec": Object { - "column": "timestamp", - "format": "iso", - }, - }, - "ioConfig": Object { - "inputFormat": Object { - "type": "json", - }, - "inputSource": Object { - "type": "http", - "uris": Array [ - "https://static.imply.io/data/wikipedia.json.gz", - ], - }, - "type": "index_parallel", - }, - "tuningConfig": Object { - "type": "index_parallel", - }, - }, - "type": "index_parallel", - } - `); - }); -}); -describe('druid-query.ts', () => { - it('spec-utils parseHtmlError', () => { - expect(parseHtmlError('
')).toMatchInlineSnapshot(`undefined`); - }); - - it('spec-utils parseHtmlError', () => { - expect(getDruidErrorMessage({})).toMatchInlineSnapshot(`undefined`); - }); - - it('spec-utils parseQueryPlan', () => { - expect(parseQueryPlan('start')).toMatchInlineSnapshot(`"start"`); - }); }); diff --git a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap index 64b412dc15a..947e1d2bc32 100755 --- a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap +++ b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap @@ -64,6 +64,7 @@ exports[`data source view matches snapshot 1`] = ` "Segment load/drop queues", "Total data size", "Segment size", + "Segment granularity", "Total rows", "Avg. row size", "Replicated size", @@ -193,6 +194,19 @@ exports[`data source view matches snapshot 1`] = ` "show": true, "width": 220, }, + Object { + "Cell": [Function], + "Header": + Segment +
+ granularity + , + "accessor": [Function], + "filterable": false, + "id": "segment_granularity", + "show": true, + "width": 100, + }, Object { "Cell": [Function], "Header": diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx index 9ad70d9d8cb..effd3d1301c 100644 --- a/web-console/src/views/datasource-view/datasource-view.tsx +++ b/web-console/src/views/datasource-view/datasource-view.tsx @@ -39,14 +39,20 @@ import { } from '../../components'; import { AsyncActionDialog, CompactionDialog, RetentionDialog } from '../../dialogs'; import { DatasourceTableActionDialog } from '../../dialogs/datasource-table-action-dialog/datasource-table-action-dialog'; +import { + CompactionConfig, + CompactionStatus, + formatCompactionConfigAndStatus, + zeroCompactionStatus, +} from '../../druid-models'; import { AppToaster } from '../../singletons/toaster'; import { addFilter, - CompactionConfig, - CompactionStatus, + Capabilities, + CapabilitiesMode, countBy, + deepGet, formatBytes, - formatCompactionConfigAndStatus, formatInteger, formatMillions, formatPercent, @@ -57,13 +63,10 @@ import { queryDruidSql, QueryManager, QueryState, - zeroCompactionStatus, } from '../../utils'; import { BasicAction } from '../../utils/basic-action'; -import { Capabilities, CapabilitiesMode } from '../../utils/capabilities'; import { Rule, RuleUtil } from '../../utils/load-rule'; import { LocalStorageBackedArray } from '../../utils/local-storage-backed-array'; -import { deepGet } from '../../utils/object-change'; import './datasource-view.scss'; @@ -74,6 +77,7 @@ const tableColumns: Record = { 'Segment load/drop queues', 'Total data size', 'Segment size', + 'Segment granularity', 'Total rows', 'Avg. row size', 'Replicated size', @@ -100,6 +104,7 @@ const tableColumns: Record = { 'Segment load/drop queues', 'Total data size', 'Segment size', + 'Segment granularity', 'Total rows', 'Avg. row size', 'Replicated size', @@ -149,6 +154,11 @@ interface DatasourceQueryResultRow { readonly num_available_segments: number; readonly num_segments_to_load: number; readonly num_segments_to_drop: number; + readonly minute_aligned_segments: number; + readonly hour_aligned_segments: number; + readonly day_aligned_segments: number; + readonly month_aligned_segments: number; + readonly year_aligned_segments: number; readonly total_data_size: number; readonly replicated_size: number; readonly min_segment_rows: number; @@ -158,6 +168,17 @@ interface DatasourceQueryResultRow { readonly avg_row_size: number; } +function segmentGranularityCountsToRank(row: DatasourceQueryResultRow): number { + return ( + Number(Boolean(row.num_segments)) + + Number(Boolean(row.minute_aligned_segments)) + + Number(Boolean(row.hour_aligned_segments)) + + Number(Boolean(row.day_aligned_segments)) + + Number(Boolean(row.month_aligned_segments)) + + Number(Boolean(row.year_aligned_segments)) + ); +} + interface Datasource extends DatasourceQueryResultRow { readonly rules: Rule[]; readonly compactionConfig?: CompactionConfig; @@ -227,6 +248,11 @@ export class DatasourcesView extends React.PureComponent< COUNT(*) FILTER (WHERE is_available = 1 AND ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_available_segments, COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load, COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop, + COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments, + COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments, + COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments, + COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments, + COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments, SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size, SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size, MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows, @@ -306,6 +332,11 @@ GROUP BY 1`; num_segments: numSegments, num_segments_to_load: segmentsToLoad, num_segments_to_drop: 0, + minute_aligned_segments: -1, + hour_aligned_segments: -1, + day_aligned_segments: -1, + month_aligned_segments: -1, + year_aligned_segments: -1, replicated_size: -1, total_data_size: totalDataSize, min_segment_rows: -1, @@ -1031,6 +1062,37 @@ GROUP BY 1`; ), }, + { + Header: twoLines('Segment', 'granularity'), + show: capabilities.hasSql() && hiddenColumns.exists('Segment granularity'), + id: 'segment_granularity', + accessor: segmentGranularityCountsToRank, + filterable: false, + width: 100, + Cell: ({ original }) => { + const segmentGranularities: string[] = []; + if (!original.num_segments) return '-'; + if (original.num_segments - original.minute_aligned_segments) { + segmentGranularities.push('Sub minute'); + } + if (original.minute_aligned_segments - original.hour_aligned_segments) { + segmentGranularities.push('Minute'); + } + if (original.hour_aligned_segments - original.day_aligned_segments) { + segmentGranularities.push('Hour'); + } + if (original.day_aligned_segments - original.month_aligned_segments) { + segmentGranularities.push('Day'); + } + if (original.month_aligned_segments - original.year_aligned_segments) { + segmentGranularities.push('Month'); + } + if (original.year_aligned_segments) { + segmentGranularities.push('Year'); + } + return segmentGranularities.join(', '); + }, + }, { Header: twoLines('Total', 'rows'), show: capabilities.hasSql() && hiddenColumns.exists('Total rows'), diff --git a/web-console/src/views/home-view/segments-card/segments-card.tsx b/web-console/src/views/home-view/segments-card/segments-card.tsx index d84061e30b9..2245aae8d08 100644 --- a/web-console/src/views/home-view/segments-card/segments-card.tsx +++ b/web-console/src/views/home-view/segments-card/segments-card.tsx @@ -22,9 +22,7 @@ import { sum } from 'd3-array'; import React from 'react'; import { useQueryManager } from '../../../hooks'; -import { pluralIfNeeded, queryDruidSql } from '../../../utils'; -import { Capabilities } from '../../../utils/capabilities'; -import { deepGet } from '../../../utils/object-change'; +import { Capabilities, deepGet, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface SegmentCounts { diff --git a/web-console/src/views/ingestion-view/ingestion-view.spec.tsx b/web-console/src/views/ingestion-view/ingestion-view.spec.tsx index fa8c867e18f..ddbaa9d9103 100644 --- a/web-console/src/views/ingestion-view/ingestion-view.spec.tsx +++ b/web-console/src/views/ingestion-view/ingestion-view.spec.tsx @@ -32,7 +32,6 @@ describe('tasks view', () => { datasourceId={'datasource'} goToDatasource={() => {}} goToQuery={() => {}} - goToMiddleManager={() => {}} goToLoadData={() => {}} capabilities={Capabilities.FULL} />, diff --git a/web-console/src/views/ingestion-view/ingestion-view.tsx b/web-console/src/views/ingestion-view/ingestion-view.tsx index 380c1cf0a29..d4826d8105d 100644 --- a/web-console/src/views/ingestion-view/ingestion-view.tsx +++ b/web-console/src/views/ingestion-view/ingestion-view.tsx @@ -45,11 +45,13 @@ import { addFilter, addFilterRaw, booleanCustomTableFilter, + deepGet, formatDuration, getDruidErrorMessage, localStorageGet, LocalStorageKeys, localStorageSet, + oneOf, queryDruidSql, QueryManager, QueryState, @@ -57,7 +59,6 @@ import { import { BasicAction } from '../../utils/basic-action'; import { Capabilities } from '../../utils/capabilities'; import { LocalStorageBackedArray } from '../../utils/local-storage-backed-array'; -import { deepGet } from '../../utils/object-change'; import './ingestion-view.scss'; @@ -108,7 +109,6 @@ export interface IngestionViewProps { openDialog: string | undefined; goToDatasource: (datasource: string) => void; goToQuery: (initSql: string) => void; - goToMiddleManager: (middleManager: string) => void; goToLoadData: (supervisorId?: string, taskId?: string) => void; capabilities: Capabilities; } @@ -385,7 +385,7 @@ ORDER BY "rank" DESC, "created_time" DESC`; const { goToDatasource, goToLoadData } = this.props; const actions: BasicAction[] = []; - if (type === 'kafka' || type === 'kinesis') { + if (oneOf(type, 'kafka', 'kinesis')) { actions.push( { icon: IconNames.MULTI_SELECT, @@ -659,14 +659,14 @@ ORDER BY "rank" DESC, "created_time" DESC`; onAction: () => goToDatasource(datasource), }); } - if (type === 'index' || type === 'index_parallel') { + if (oneOf(type, 'index', 'index_parallel')) { actions.push({ icon: IconNames.CLOUD_UPLOAD, title: 'Open in data loader', onAction: () => goToLoadData(undefined, id), }); } - if (status === 'RUNNING' || status === 'WAITING' || status === 'PENDING') { + if (oneOf(status, 'RUNNING', 'WAITING', 'PENDING')) { actions.push({ icon: IconNames.CROSS, title: 'Kill', @@ -704,7 +704,6 @@ ORDER BY "rank" DESC, "created_time" DESC`; } renderTaskTable() { - const { goToMiddleManager } = this.props; const { tasksState, taskFilter, @@ -812,21 +811,12 @@ ORDER BY "rank" DESC, "created_time" DESC`; }), Cell: row => { if (row.aggregated) return ''; - const { status, location } = row.original; - const locationHostname = location ? location.split(':')[0] : null; + const { status } = row.original; const errorMsg = row.original.error_msg; return ( ●  {status} - {location && ( - goToMiddleManager(locationHostname)} - title={`Go to: ${locationHostname}`} - > -  ➚ - - )} {errorMsg && ( this.setState({ alertErrorMsg: errorMsg })} diff --git a/web-console/src/views/load-data-view/filter-table/filter-table.tsx b/web-console/src/views/load-data-view/filter-table/filter-table.tsx index 89f380f44c7..db0bddc3eed 100644 --- a/web-console/src/views/load-data-view/filter-table/filter-table.tsx +++ b/web-console/src/views/load-data-view/filter-table/filter-table.tsx @@ -21,8 +21,8 @@ import React from 'react'; import ReactTable from 'react-table'; import { TableCell } from '../../../components'; +import { DruidFilter } from '../../../druid-models'; import { caseInsensitiveContains, filterMap } from '../../../utils'; -import { DruidFilter } from '../../../utils/ingestion-spec'; import { HeaderAndRows, SampleEntry } from '../../../utils/sampler'; import './filter-table.scss'; diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss index 905352ea096..bbc8627e8d1 100644 --- a/web-console/src/views/load-data-view/load-data-view.scss +++ b/web-console/src/views/load-data-view/load-data-view.scss @@ -19,6 +19,12 @@ @import '~@blueprintjs/core/src/common/colors'; @import '../../variables'; +$control-bar-width: 300px; + +$icon-width: 100px; +$actual-icon-width: 520px; +$actual-icon-height: 400px; + @mixin sunk-panel { background: rgba($dark-gray1, 0.5); border-radius: $pt-border-radius; @@ -30,7 +36,7 @@ height: 100%; display: grid; grid-gap: $thin-padding 5px; - grid-template-columns: 1fr 280px; + grid-template-columns: 1fr $control-bar-width; grid-template-rows: 60px 1fr 28px; grid-template-areas: 'navi navi' @@ -133,7 +139,8 @@ } img { - width: 100px; + width: $icon-width; + height: $icon-width * ($actual-icon-height / $actual-icon-width); display: inline-block; } } @@ -144,7 +151,7 @@ &.tuning, &.publish { grid-gap: 20px 40px; - grid-template-columns: 1fr 1fr 280px; + grid-template-columns: 1fr 1fr $control-bar-width; grid-template-areas: 'navi navi navi' 'main othr ctrl' diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 2d5074a752a..4db8286bf53 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -54,53 +54,46 @@ import { } from '../../components'; import { FormGroupWithInfo } from '../../components/form-group-with-info/form-group-with-info'; import { AsyncActionDialog } from '../../dialogs'; -import { getLink } from '../../links'; -import { AppToaster } from '../../singletons/toaster'; -import { UrlBaser } from '../../singletons/url-baser'; import { - filterMap, - getDruidErrorMessage, - localStorageGet, - LocalStorageKeys, - localStorageSet, - parseJson, - pluralIfNeeded, - QueryState, -} from '../../utils'; -import { NUMERIC_TIME_FORMATS, possibleDruidFormatForValues } from '../../utils/druid-time'; -import { updateSchemaWithSample } from '../../utils/druid-type'; + addTimestampTransform, + CONSTANT_TIMESTAMP_SPEC, + CONSTANT_TIMESTAMP_SPEC_FIELDS, + DIMENSION_SPEC_FIELDS, + FILTER_FIELDS, + FLATTEN_FIELD_FIELDS, + getTimestampExpressionFields, + getTimestampSchema, + INPUT_FORMAT_FIELDS, + METRIC_SPEC_FIELDS, + removeTimestampTransform, + TIMESTAMP_SPEC_FIELDS, + TimestampSpec, + Transform, + TRANSFORM_FIELDS, + updateSchemaWithSample, +} from '../../druid-models'; import { - adjustIngestionSpec, adjustTuningConfig, cleanSpec, + computeFlattenPathsForData, DimensionMode, DimensionSpec, DimensionsSpec, DruidFilter, - EMPTY_ARRAY, - EMPTY_OBJECT, fillDataSourceNameIfNeeded, fillInputFormat, FlattenField, - getConstantTimestampSpec, getDimensionMode, - getDimensionSpecFormFields, - getFilterFormFields, - getFlattenFieldFormFields, getIngestionComboType, getIngestionDocLink, getIngestionImage, getIngestionTitle, - getInputFormatFormFields, getIoConfigFormFields, getIoConfigTuningFormFields, - getMetricSpecFormFields, getPartitionRelatedTuningSpecFormFields, getRequiredModule, getRollup, getSpecType, - getTimestampSpecFormFields, - getTransformFormFields, getTuningSpecFormFields, GranularitySpec, IngestionComboTypeWithExtra, @@ -110,7 +103,6 @@ import { invalidIoConfig, invalidTuningConfig, IoConfig, - isColumnTimestampSpec, isDruidSource, isEmptyIngestionSpec, issueWithIoConfig, @@ -119,14 +111,33 @@ import { MAX_INLINE_DATA_LENGTH, MetricSpec, normalizeSpec, + NUMERIC_TIME_FORMATS, + possibleDruidFormatForValues, splitFilter, - TimestampSpec, - Transform, TuningConfig, updateIngestionType, upgradeSpec, -} from '../../utils/ingestion-spec'; -import { deepDelete, deepGet, deepSet } from '../../utils/object-change'; +} from '../../druid-models'; +import { getLink } from '../../links'; +import { AppToaster } from '../../singletons/toaster'; +import { UrlBaser } from '../../singletons/url-baser'; +import { + deepDelete, + deepGet, + deepSet, + deepSetMulti, + EMPTY_ARRAY, + EMPTY_OBJECT, + filterMap, + getDruidErrorMessage, + localStorageGet, + LocalStorageKeys, + localStorageSet, + oneOf, + parseJson, + pluralIfNeeded, + QueryState, +} from '../../utils'; import { CacheRows, ExampleManifest, @@ -146,7 +157,6 @@ import { SampleResponseWithExtraInfo, SampleStrategy, } from '../../utils/sampler'; -import { computeFlattenPathsForData } from '../../utils/spec-utils'; import { ExamplePicker } from './example-picker/example-picker'; import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table'; @@ -187,7 +197,7 @@ function showBlankLine(line: SampleEntry): string { } function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec { - if (!headerAndRows) return getConstantTimestampSpec(); + if (!headerAndRows) return CONSTANT_TIMESTAMP_SPEC; const timestampSpecs = filterMap(headerAndRows.header, sampleHeader => { const possibleFormat = possibleDruidFormatForValues( @@ -204,7 +214,7 @@ function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec { timestampSpecs.find(ts => /time/i.test(ts.column)) || // Use a suggestion that has time in the name if possible timestampSpecs.find(ts => !NUMERIC_TIME_FORMATS.includes(ts.format)) || // Use a suggestion that is not numeric timestampSpecs[0] || // Fall back to the first one - getConstantTimestampSpec() // Ok, empty it is... + CONSTANT_TIMESTAMP_SPEC // Ok, empty it is... ); } @@ -300,7 +310,7 @@ export interface LoadDataViewState { // for timestamp timestampQueryState: QueryState<{ headerAndRows: HeaderAndRows; - timestampSpec: TimestampSpec; + spec: IngestionSpec; }>; // for transform @@ -454,7 +464,6 @@ export class LoadDataView extends React.PureComponent { newSpec = normalizeSpec(newSpec); newSpec = upgradeSpec(newSpec); - newSpec = adjustIngestionSpec(newSpec); const deltaState: Partial = { spec: newSpec, specPreview: newSpec }; if (!deepGet(newSpec, 'spec.ioConfig.type')) { deltaState.cacheRows = undefined; @@ -470,7 +479,7 @@ export class LoadDataView extends React.PureComponent { this.setState(state => { localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(state.specPreview)); - return { spec: state.specPreview }; + return { spec: Object.assign({}, state.specPreview) }; }); }; @@ -577,14 +586,15 @@ export class LoadDataView extends React.PureComponent) { const previewSpecSame = this.isPreviewSpecSame(); + const queryStateHasError = Boolean(queryState && queryState.error); return (