From 868bb42301460570b6ccfcbd4d251d68cd326da1 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Sat, 21 Sep 2019 21:00:21 -0700 Subject: [PATCH] clean up default values and add infos (#8567) --- .../__snapshots__/auto-form.spec.tsx.snap | 2 +- .../src/components/auto-form/auto-form.tsx | 11 +- .../compaction-dialog.spec.tsx.snap | 220 +++++++++++++++++- .../compaction-dialog/compaction-dialog.tsx | 85 +++++-- .../coordinator-dynamic-config-dialog.tsx | 14 +- .../overlord-dynamic-config-dialog.tsx | 10 +- web-console/src/utils/ingestion-spec.tsx | 18 +- .../load-data-view.spec.tsx.snap | 2 +- .../views/load-data-view/load-data-view.tsx | 5 +- 9 files changed, 314 insertions(+), 53 deletions(-) diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap index 7601f534a17..99c41445d10 100644 --- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap +++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap @@ -105,7 +105,7 @@ exports[`auto-form snapshot matches snapshot 1`] = ` class="bp3-form-content" >
{ model: T | undefined; onChange: (newModel: T) => void; showCustom?: (model: T) => boolean; - updateJSONValidity?: (jsonValidity: boolean) => void; + updateJsonValidity?: (jsonValidity: boolean) => void; large?: boolean; } @@ -169,6 +169,7 @@ export class AutoForm> extends React.PureComponent min={0} stepSize={1000} majorStepSize={1000000} + fill large={large} disabled={AutoForm.evaluateFunctor(field.disabled, model)} /> @@ -235,19 +236,19 @@ export class AutoForm> extends React.PureComponent } private renderJSONInput(field: Field): JSX.Element { - const { model, updateJSONValidity } = this.props; + const { model, updateJsonValidity } = this.props; const { jsonInputsValidity } = this.state; const updateInputValidity = (e: any) => { - if (updateJSONValidity) { + if (updateJsonValidity) { const newJSONInputValidity = Object.assign({}, jsonInputsValidity, { [field.name]: e }); this.setState({ jsonInputsValidity: newJSONInputValidity, }); - const allJSONValid: boolean = Object.keys(newJSONInputValidity).every( + const allJsonValid: boolean = Object.keys(newJSONInputValidity).every( property => newJSONInputValidity[property] === true, ); - updateJSONValidity(allJSONValid); + updateJsonValidity(allJsonValid); } }; diff --git a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap index 3034ed409d9..fd23131e9b1 100644 --- a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap @@ -64,7 +64,35 @@ exports[`compaction dialog matches snapshot 1`] = ` + > + + + + + + info-sign + + + + + + +
+ > + + + + + + info-sign + + + + + + +
+ > + + + + + + info-sign + + + + + + +
@@ -261,7 +345,35 @@ exports[`compaction dialog matches snapshot 1`] = ` + > + + + + + + info-sign + + + + + + +
+ > + + + + + + info-sign + + + + + + +
+ > + + + + + + info-sign + + + + + + +
+ > + + + + + + info-sign + + + + + + +
; - allJSONValid: boolean; + allJsonValid: boolean; } export class CompactionDialog extends React.PureComponent< @@ -45,27 +45,17 @@ export class CompactionDialog extends React.PureComponent< constructor(props: CompactionDialogProps) { super(props); this.state = { - allJSONValid: true, + allJsonValid: true, }; } componentDidMount(): void { const { datasource, compactionConfig } = this.props; - let config: Record = { - dataSource: datasource, - inputSegmentSizeBytes: 419430400, - maxNumSegmentsToCompact: 150, - skipOffsetFromLatest: 'P1D', - targetCompactionSizeBytes: CompactionDialog.DEFAULT_TARGET_COMPACTION_SIZE_BYTES, - taskContext: null, - taskPriority: 25, - tuningConfig: null, - }; - if (compactionConfig !== undefined) { - config = compactionConfig; - } + this.setState({ - currentConfig: config, + currentConfig: compactionConfig || { + dataSource: datasource, + }, }); } @@ -73,12 +63,14 @@ export class CompactionDialog extends React.PureComponent< const { onSave } = this.props; const { currentConfig } = this.state; if (!currentConfig) return; + onSave(currentConfig); }; render(): JSX.Element { const { onClose, onDelete, datasource, compactionConfig } = this.props; - const { currentConfig, allJSONValid } = this.state; + const { currentConfig, allJsonValid } = this.state; + return ( + Maximum number of total segment bytes processed per compaction task. Since a time + chunk must be processed in its entirety, if the segments for a particular time + chunk have a total size in bytes greater than this parameter, compaction will not + run for that time chunk. Because each compaction task runs with a single thread, + setting this value too far above 1–2GB will result in compaction tasks taking an + excessive amount of time. +

+ ), }, { name: 'maxNumSegmentsToCompact', type: 'number', + defaultValue: 150, + info: ( +

+ Maximum number of segments to compact together per compaction task. Since a time + chunk must be processed in its entirety, if a time chunk has a total number of + segments greater than this parameter, compaction will not run for that time chunk. +

+ ), }, { name: 'skipOffsetFromLatest', type: 'string', + defaultValue: 'P1D', + info: ( +

+ The offset for searching segments to be compacted. Strongly recommended to set for + realtime dataSources. +

+ ), }, { name: 'targetCompactionSizeBytes', type: 'number', + defaultValue: CompactionDialog.DEFAULT_TARGET_COMPACTION_SIZE_BYTES, + info: ( +

+ The target segment size, for each segment, after compaction. The actual sizes of + compacted segments might be slightly larger or smaller than this value. Each + compaction task may generate more than one output segment, and it will try to keep + each output segment close to this configured size. +

+ ), }, { name: 'taskContext', type: 'json', + info: ( +

+ + Task context + {' '} + for compaction tasks. +

+ ), }, { name: 'taskPriority', type: 'number', + defaultValue: 25, + info:

Priority of the compaction task.

, }, { name: 'tuningConfig', type: 'json', + info: ( +

+ + Tuning config + {' '} + for compaction tasks. +

+ ), }, ]} model={currentConfig} onChange={m => this.setState({ currentConfig: m })} - updateJSONValidity={e => this.setState({ allJSONValid: e })} + updateJsonValidity={e => this.setState({ allJsonValid: e })} />
@@ -135,7 +180,7 @@ export class CompactionDialog extends React.PureComponent< text="Submit" intent={Intent.PRIMARY} onClick={this.handleSubmit} - disabled={!currentConfig || !allJSONValid} + disabled={!currentConfig || !allJsonValid} />
diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx index 181eb27d90e..c7cc4075907 100644 --- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx @@ -133,6 +133,12 @@ export class CoordinatorDynamicConfigDialog extends React.PureComponent<

The maximum number of segments that can be moved at any given time., + }, { name: 'balancerComputeThreads', type: 'number', @@ -148,6 +154,7 @@ export class CoordinatorDynamicConfigDialog extends React.PureComponent< { name: 'emitBalancingStats', type: 'boolean', + defaultValue: false, info: ( <> Boolean flag for whether or not we should emit balancing stats. This is an @@ -158,6 +165,7 @@ export class CoordinatorDynamicConfigDialog extends React.PureComponent< { name: 'killAllDataSources', type: 'boolean', + defaultValue: false, info: ( <> Send kill tasks for ALL dataSources if property{' '} @@ -205,12 +213,6 @@ export class CoordinatorDynamicConfigDialog extends React.PureComponent< ), }, - { - name: 'maxSegmentsToMove', - type: 'number', - defaultValue: 5, - info: <>The maximum number of segments that can be moved at any given time., - }, { name: 'mergeBytesLimit', type: 'size-bytes', diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx index b94d0d8b76a..b46e8637130 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx @@ -34,7 +34,7 @@ export interface OverlordDynamicConfigDialogProps { export interface OverlordDynamicConfigDialogState { dynamicConfig?: Record; - allJSONValid: boolean; + allJsonValid: boolean; historyRecords: any[]; } @@ -47,7 +47,7 @@ export class OverlordDynamicConfigDialog extends React.PureComponent< constructor(props: OverlordDynamicConfigDialogProps) { super(props); this.state = { - allJSONValid: true, + allJsonValid: true, historyRecords: [], }; @@ -115,7 +115,7 @@ export class OverlordDynamicConfigDialog extends React.PureComponent< render(): JSX.Element { const { onClose } = this.props; - const { dynamicConfig, allJSONValid, historyRecords } = this.state; + const { dynamicConfig, allJsonValid, historyRecords } = this.state; return (

@@ -147,7 +147,7 @@ export class OverlordDynamicConfigDialog extends React.PureComponent< ]} model={dynamicConfig} onChange={m => this.setState({ dynamicConfig: m })} - updateJSONValidity={e => this.setState({ allJSONValid: e })} + updateJsonValidity={e => this.setState({ allJsonValid: e })} /> ); diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx index 1ab0d8de5ea..fdc85a6c645 100644 --- a/web-console/src/utils/ingestion-spec.tsx +++ b/web-console/src/utils/ingestion-spec.tsx @@ -787,6 +787,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F type: 'string-array', placeholder: 'https://example.com/path/to/file1.ext, https://example.com/path/to/file2.ext', + required: true, info: (

The full URI of your file. To ingest from multiple URIs, use commas to separate each @@ -818,6 +819,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F label: 'Base directory', type: 'string', placeholder: '/path/to/files/', + required: true, info: ( <> @@ -831,7 +833,8 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F name: 'firehose.filter', label: 'File filter', type: 'string', - defaultValue: '*.*', + required: true, + suggestions: ['*', '*.json', '*.json.gz', '*.csv', '*.tsv'], info: ( <> @@ -856,6 +859,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F name: 'firehose.dataSource', label: 'Datasource', type: 'string', + required: true, info:

The datasource to fetch rows from.

, }, { @@ -868,6 +872,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F `${CURRENT_YEAR}-01-01/${CURRENT_YEAR + 1}-01-01`, `${CURRENT_YEAR}/${CURRENT_YEAR + 1}`, ], + required: true, info: (

A String representing ISO-8601 Interval. This defines the time range to fetch the data @@ -930,6 +935,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F label: 'S3 URIs', type: 'string-array', placeholder: 's3://your-bucket/some-file1.ext, s3://your-bucket/some-file2.ext', + required: true, defined: ioConfig => !deepGet(ioConfig, 'firehose.prefixes'), info: ( <> @@ -946,6 +952,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F label: 'S3 prefixes', type: 'string-array', placeholder: 's3://your-bucket/some-path1, s3://your-bucket/some-path2', + required: true, defined: ioConfig => !deepGet(ioConfig, 'firehose.uris'), info: ( <> @@ -963,6 +970,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F name: 'firehose.blobs', label: 'Google blobs', type: 'json', + required: true, info: ( <>

@@ -983,6 +991,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F name: 'consumerProperties.{bootstrap.servers}', label: 'Bootstrap servers', type: 'string', + required: true, info: ( <> @@ -998,6 +1007,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F { name: 'topic', type: 'string', + required: true, defined: (i: IoConfig) => i.type === 'kafka', }, { @@ -1021,6 +1031,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F name: 'stream', type: 'string', placeholder: 'your-kinesis-stream', + required: true, info: <>The Kinesis stream to read., }, { @@ -1051,6 +1062,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F 'kinesis.us-gov-east-1.amazonaws.com', 'kinesis.us-gov-west-1.amazonaws.com', ], + required: true, info: ( <> The Amazon Kinesis stream endpoint for a region. You can find a list of endpoints{' '} @@ -1987,6 +1999,10 @@ export function updateIngestionType( if (firehoseType) { newSpec = deepSet(newSpec, 'ioConfig.firehose', { type: firehoseType }); + + if (firehoseType === 'local') { + newSpec = deepSet(newSpec, 'ioConfig.firehose.filter', '*'); + } } if (!deepGet(spec, 'dataSchema.dataSource')) { diff --git a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap index ae33b658d0f..ff459c7e179 100644 --- a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap +++ b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap @@ -158,7 +158,7 @@ exports[`load data view matches snapshot 1`] = ` icon="manually-entered-data" key="spec" onClick={[Function]} - text="Edit JSON spec" + text="Edit spec" />

diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 17078fdb5a2..5d1a6358f35 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -232,7 +232,7 @@ const VIEW_TITLE: Record = { partition: 'Partition', tuning: 'Tune', publish: 'Publish', - spec: 'Edit JSON spec', + spec: 'Edit spec', loading: 'Loading', }; @@ -2788,6 +2788,7 @@ export class LoadDataView extends React.PureComponent !deepGet(spec, 'tuningConfig.forceGuaranteedRollup'), info: ( <> @@ -2809,8 +2810,8 @@ export class LoadDataView extends React.PureComponent If true, log an error message when a parsing exception occurs, containing