diff --git a/web-console/assets/druid.png b/web-console/assets/druid.png new file mode 100644 index 00000000000..b494638ee31 Binary files /dev/null and b/web-console/assets/druid.png differ diff --git a/web-console/assets/example.png b/web-console/assets/example.png new file mode 100644 index 00000000000..faf96784f2a Binary files /dev/null and b/web-console/assets/example.png differ diff --git a/web-console/assets/hadoop.png b/web-console/assets/hadoop.png new file mode 100644 index 00000000000..9b69d38c5d4 Binary files /dev/null and b/web-console/assets/hadoop.png differ diff --git a/web-console/assets/http.png b/web-console/assets/http.png new file mode 100644 index 00000000000..ee041ba8785 Binary files /dev/null and b/web-console/assets/http.png differ diff --git a/web-console/assets/kafka.png b/web-console/assets/kafka.png new file mode 100644 index 00000000000..0f0eb7c3320 Binary files /dev/null and b/web-console/assets/kafka.png differ diff --git a/web-console/assets/kinesis.png b/web-console/assets/kinesis.png new file mode 100644 index 00000000000..d87a3cb1e36 Binary files /dev/null and b/web-console/assets/kinesis.png differ diff --git a/web-console/assets/local.png b/web-console/assets/local.png new file mode 100644 index 00000000000..25c523bdacf Binary files /dev/null and b/web-console/assets/local.png differ diff --git a/web-console/assets/other.png b/web-console/assets/other.png new file mode 100644 index 00000000000..b2b977d1168 Binary files /dev/null and b/web-console/assets/other.png differ diff --git a/web-console/assets/static-google-blobstore.png b/web-console/assets/static-google-blobstore.png new file mode 100644 index 00000000000..960cbcfd255 Binary files /dev/null and b/web-console/assets/static-google-blobstore.png differ diff --git a/web-console/assets/static-s3.png b/web-console/assets/static-s3.png new file mode 100644 index 00000000000..e73112e99e8 Binary files /dev/null and b/web-console/assets/static-s3.png differ diff --git a/web-console/package-lock.json b/web-console/package-lock.json index fde730535dc..7f7c8046be2 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -229,9 +229,9 @@ } }, "@blueprintjs/core": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.15.1.tgz", - "integrity": "sha512-M8ltbqqlMZuZ6SEuqo/3Fr59ZcUfd8Er7ocbm7EACVfRW7dRhOCd/TKkf2kfICNtCDwznwXk0iAePLXZhUGtQg==", + "version": "3.16.2", + "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.16.2.tgz", + "integrity": "sha512-u+mSITWaNDwbdaPrbKx9XyxGsF4725SCAidWjd367ysX7AxCo4PK4SsFQVfXNylXpVWHQhJZekuo7+hdksc9lA==", "requires": { "@blueprintjs/icons": "^3.8.0", "@types/dom4": "^2.0.1", @@ -246,9 +246,9 @@ } }, "@blueprintjs/icons": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.8.0.tgz", - "integrity": "sha512-yHaRQ3vfV9Gf3foZ4ONtxddz+u5ufkHqHj8Ia5VhPbFgG4el+cPdmsGGIIM72rgKS1KQa5Ay+ggjpByUlXvrKg==", + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.9.0.tgz", + "integrity": "sha512-kq1Bh6PtOF4PcuxcDme8NmnSlkfO0IV89FriZGo6zSA1+OOzSwzvoKqa6S7vJe8xCPPLO5r7lE9AjeOuGeH97g==", "requires": { "classnames": "^2.2", "tslib": "^1.9.0" diff --git a/web-console/package.json b/web-console/package.json index 3e07b65a7ce..a4369d67e2c 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -50,7 +50,8 @@ "stylelint": "stylelint 'src/**/*.scss'" }, "dependencies": { - "@blueprintjs/core": "^3.15.1", + "@blueprintjs/core": "^3.16.2", + "@blueprintjs/icons": "^3.9.0", "@types/memoize-one": "^4.1.1", "axios": "^0.19.0", "brace": "^0.11.1", diff --git a/web-console/script/cp-to b/web-console/script/cp-to index b8ad2ebeb4d..3943a10daf4 100755 --- a/web-console/script/cp-to +++ b/web-console/script/cp-to @@ -25,3 +25,4 @@ cp -r coordinator-console "$1" cp -r old-console "$1" cp -r pages "$1" cp -r public "$1" +cp -r assets "$1" diff --git a/web-console/script/create-sql-function-doc.js b/web-console/script/create-sql-function-doc.js index 51cac06655e..f8dd6380a38 100755 --- a/web-console/script/create-sql-function-doc.js +++ b/web-console/script/create-sql-function-doc.js @@ -34,27 +34,31 @@ const readDoc = async () => { if (functionMatch) { functionDocs.push({ syntax: functionMatch[1], - description: functionMatch[2] - }) + description: functionMatch[2], + }); } const dataTypeMatch = line.match(/^\|([A-Z]+)\|([A-Z]+)\|(.*)\|(.*)\|$/); if (dataTypeMatch) { dataTypeDocs.push({ syntax: dataTypeMatch[1], - description: dataTypeMatch[4] || `Druid runtime type: ${dataTypeMatch[2]}` - }) + description: dataTypeMatch[4] || `Druid runtime type: ${dataTypeMatch[2]}`, + }); } } // Make sure there are at least 10 functions for sanity if (functionDocs.length < 10) { - throw new Error(`Did not find enough function entries did the structure of '${readfile}' change? (found ${functionDocs.length})`); + throw new Error( + `Did not find enough function entries did the structure of '${readfile}' change? (found ${functionDocs.length})`, + ); } // Make sure there are at least 5 data types for sanity if (dataTypeDocs.length < 10) { - throw new Error(`Did not find enough data type entries did the structure of '${readfile}' change? (found ${dataTypeDocs.length})`); + throw new Error( + `Did not find enough data type entries did the structure of '${readfile}' change? (found ${dataTypeDocs.length})`, + ); } const content = `/* diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx index 84fa63c40b1..dc115cf4424 100644 --- a/web-console/src/utils/ingestion-spec.tsx +++ b/web-console/src/utils/ingestion-spec.tsx @@ -37,6 +37,10 @@ export interface IngestionSpec { tuningConfig?: TuningConfig; } +export function isEmptyIngestionSpec(spec: IngestionSpec) { + return Object.keys(spec).length === 0; +} + export type IngestionType = 'kafka' | 'kinesis' | 'index_hadoop' | 'index' | 'index_parallel'; // A combination of IngestionType and firehose @@ -48,6 +52,9 @@ export type IngestionComboType = | 'index:static-s3' | 'index:static-google-blobstore'; +// Some extra values that can be selected in the initial screen +export type IngestionComboTypeWithExtra = IngestionComboType | 'hadoop' | 'example' | 'other'; + function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): string { switch (ingestionType) { case 'kafka': @@ -87,6 +94,65 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType | return null; } +export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): string { + switch (ingestionType) { + case 'index:local': + return 'Local disk'; + + case 'index:http': + return 'HTTP(s)'; + + case 'index:static-s3': + return 'Amazon S3'; + + case 'index:static-google-blobstore': + return 'Google Cloud Storage'; + + case 'kafka': + return 'Apache Kafka'; + + case 'kinesis': + return 'Amazon Kinesis'; + + case 'hadoop': + return 'HDFS'; + + case 'example': + return 'Example data'; + + case 'other': + return 'Other'; + + default: + return 'Unknown ingestion'; + } +} + +export function getIngestionImage(ingestionType: IngestionComboTypeWithExtra): string { + const parts = ingestionType.split(':'); + if (parts.length === 2) return parts[1]; + return ingestionType; +} + +export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): string | null { + switch (ingestionType) { + case 'index:static-s3': + return 'druid-s3-extensions'; + + case 'index:static-google-blobstore': + return 'druid-google-extensions'; + + case 'kafka': + return 'druid-kafka-indexing-service'; + + case 'kinesis': + return 'druid-kinesis-indexing-service'; + + default: + return null; + } +} + // -------------- export interface DataSchema { @@ -138,7 +204,7 @@ export function getRollup(spec: IngestionSpec): boolean { return typeof specRollup === 'boolean' ? specRollup : true; } -export function getSpecType(spec: IngestionSpec): IngestionType | undefined { +export function getSpecType(spec: Partial): IngestionType | undefined { return ( deepGet(spec, 'type') || deepGet(spec, 'ioConfig.type') || deepGet(spec, 'tuningConfig.type') ); @@ -158,13 +224,21 @@ export function changeParallel(spec: IngestionSpec, parallel: boolean): Ingestio * Make sure that the types are set in the root, ioConfig, and tuningConfig * @param spec */ -export function normalizeSpecType(spec: IngestionSpec) { +export function normalizeSpec(spec: Partial): IngestionSpec { + if (!spec || typeof spec !== 'object') { + // This does not match the type of IngestionSpec but this dialog is robust enough to deal with anything but spec must be an object + spec = {}; + } + + // Make sure that if we actually get a task payload we extract the spec + if (typeof (spec as any).spec === 'object') spec = (spec as any).spec; + const specType = getSpecType(spec); - if (!specType) return spec; + if (!specType) return spec as IngestionSpec; if (!deepGet(spec, 'type')) spec = deepSet(spec, 'type', specType); if (!deepGet(spec, 'ioConfig.type')) spec = deepSet(spec, 'ioConfig.type', specType); if (!deepGet(spec, 'tuningConfig.type')) spec = deepSet(spec, 'tuningConfig.type', specType); - return spec; + return spec as IngestionSpec; } const PARSE_SPEC_FORM_FIELDS: Field[] = [ @@ -851,7 +925,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F ], info: ( <> - The AWS Kinesis stream endpoint for a region. You can find a list of endpoints{' '} + The Amazon Kinesis stream endpoint for a region. You can find a list of endpoints{' '} here @@ -1662,40 +1736,40 @@ export interface Bitmap { // -------------- -export function getBlankSpec(comboType: IngestionComboType): IngestionSpec { +export function updateIngestionType( + spec: IngestionSpec, + comboType: IngestionComboType, +): IngestionSpec { let [ingestionType, firehoseType] = comboType.split(':'); if (ingestionType === 'index') ingestionType = 'index_parallel'; const ioAndTuningConfigType = ingestionTypeToIoAndTuningConfigType( ingestionType as IngestionType, ); - const granularitySpec: GranularitySpec = { - type: 'uniform', - segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR', - queryGranularity: 'HOUR', - }; - - const spec: IngestionSpec = { - type: ingestionType, - dataSchema: { - dataSource: 'new-data-source', - granularitySpec, - }, - ioConfig: { - type: ioAndTuningConfigType, - }, - tuningConfig: { - type: ioAndTuningConfigType, - }, - } as any; + let newSpec = spec; + newSpec = deepSet(newSpec, 'type', ingestionType); + newSpec = deepSet(newSpec, 'ioConfig.type', ioAndTuningConfigType); + newSpec = deepSet(newSpec, 'tuningConfig.type', ioAndTuningConfigType); if (firehoseType) { - spec.ioConfig.firehose = { - type: firehoseType, - }; + newSpec = deepSet(newSpec, 'ioConfig.firehose', { type: firehoseType }); } - return spec; + if (!deepGet(spec, 'dataSchema.dataSource')) { + newSpec = deepSet(newSpec, 'dataSchema.dataSource', 'new-data-source'); + } + + if (!deepGet(spec, 'dataSchema.granularitySpec')) { + const granularitySpec: GranularitySpec = { + type: 'uniform', + segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR', + queryGranularity: 'HOUR', + }; + + newSpec = deepSet(newSpec, 'dataSchema.granularitySpec', granularitySpec); + } + + return newSpec; } export function fillParser(spec: IngestionSpec, sampleData: string[]): IngestionSpec { diff --git a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap index 829003f1ed6..9718f805f5d 100644 --- a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap +++ b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap @@ -2,41 +2,169 @@ exports[`load data view matches snapshot 1`] = `
- Please specify where your raw data is located +
+
+ Connect and parse raw data +
+ + + + + + +
+
+
+ Transform and configure schema +
+ + + + + +
+
+
+ Tune parameters +
+ + + + + +
+
+
+ Verify and submit +
+ + + +
+
- - Other (streaming) - - - Other (batch) - +

+ Please specify where your raw data is located +

+
- -

-

`; diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss index acd7ae0f3cb..847bea4693e 100644 --- a/web-console/src/views/load-data-view/load-data-view.scss +++ b/web-console/src/views/load-data-view/load-data-view.scss @@ -27,32 +27,45 @@ 'main ctrl' 'main next'; - &.init { - display: block; + &.welcome { + .main { + margin-left: -10px; - & > * { - margin-bottom: 15px; - } - - .intro { - font-size: 20px; - } - - .cards { .bp3-card { + position: relative; display: inline-block; vertical-align: top; width: 250px; height: 140px; - margin-right: 15px; + margin-left: 15px; margin-bottom: 15px; - font-size: 24px; + font-size: 16px; text-align: center; - padding-top: 47px; + + & > * { + user-select: none; + pointer-events: none; + } + + &.active::after { + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + content: ''; + border: 2px solid #48aff0; + border-radius: 2px; + } &.disabled { opacity: 0.4; } + + img { + width: 100px; + display: inline-block; + } } } } @@ -192,7 +205,7 @@ text-align: right; padding: 0 5px; - .prev { + .left { float: left; } } diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 0c3e426ca99..efc853d5186 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -25,6 +25,7 @@ import { Card, Classes, Code, + Elevation, FormGroup, H5, HTMLSelect, @@ -50,6 +51,7 @@ import { } from '../../components'; import { AsyncActionDialog } from '../../dialogs'; import { AppToaster } from '../../singletons/toaster'; +import { UrlBaser } from '../../singletons/url-baser'; import { filterMap, getDruidErrorMessage, @@ -72,18 +74,20 @@ import { fillDataSourceName, fillParser, FlattenField, - getBlankSpec, getDimensionMode, getDimensionSpecFormFields, getEmptyTimestampSpec, getFilterFormFields, getFlattenFieldFormFields, getIngestionComboType, + getIngestionImage, + getIngestionTitle, getIoConfigFormFields, getIoConfigTuningFormFields, getMetricSpecFormFields, getParseSpecFormFields, getPartitionRelatedTuningSpecFormFields, + getRequiredModule, getRollup, getSpecType, getTimestampSpecFormFields, @@ -91,16 +95,17 @@ import { getTuningSpecFormFields, GranularitySpec, hasParallelAbility, - IngestionComboType, + IngestionComboTypeWithExtra, IngestionSpec, IoConfig, isColumnTimestampSpec, + isEmptyIngestionSpec, isParallel, issueWithIoConfig, issueWithParser, joinFilter, MetricSpec, - normalizeSpecType, + normalizeSpec, Parser, ParseSpec, parseSpecHasFlatten, @@ -108,6 +113,7 @@ import { TimestampSpec, Transform, TuningConfig, + updateIngestionType, } from '../../utils/ingestion-spec'; import { deepDelete, deepGet, deepSet } from '../../utils/object-change'; import { @@ -163,6 +169,7 @@ function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec { } type Step = + | 'welcome' | 'connect' | 'parser' | 'timestamp' @@ -172,9 +179,11 @@ type Step = | 'partition' | 'tuning' | 'publish' - | 'json-spec' + | 'spec' | 'loading'; + const STEPS: Step[] = [ + 'welcome', 'connect', 'parser', 'timestamp', @@ -184,18 +193,19 @@ const STEPS: Step[] = [ 'partition', 'tuning', 'publish', - 'json-spec', + 'spec', 'loading', ]; const SECTIONS: { name: string; steps: Step[] }[] = [ - { name: 'Connect and parse raw data', steps: ['connect', 'parser', 'timestamp'] }, + { name: 'Connect and parse raw data', steps: ['welcome', 'connect', 'parser', 'timestamp'] }, { name: 'Transform and configure schema', steps: ['transform', 'filter', 'schema'] }, { name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] }, - { name: 'Verify and submit', steps: ['json-spec'] }, + { name: 'Verify and submit', steps: ['spec'] }, ]; const VIEW_TITLE: Record = { + welcome: 'Start', connect: 'Connect', parser: 'Parse data', timestamp: 'Parse time', @@ -205,7 +215,7 @@ const VIEW_TITLE: Record = { partition: 'Partition', tuning: 'Tune', publish: 'Publish', - 'json-spec': 'Edit JSON spec', + spec: 'Edit JSON spec', loading: 'Loading', }; @@ -224,9 +234,11 @@ export interface LoadDataViewState { newRollup: boolean | null; newDimensionMode: DimensionMode | null; - // general + // welcome overlordModules: string[] | null; - overlordModuleNeededMessage: string | null; + selectedComboType: IngestionComboTypeWithExtra | null; + + // general sampleStrategy: SampleStrategy; columnFilter: string; specialColumnsOnly: boolean; @@ -278,7 +290,7 @@ export class LoadDataView extends React.PureComponent { - if (!newSpec || typeof newSpec !== 'object') { - // This does not match the type of IngestionSpec but this dialog is robust enough to deal with anything but spec must be an object - newSpec = {} as any; - } + newSpec = normalizeSpec(newSpec); this.setState({ spec: newSpec }); localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(newSpec)); }; render() { - const { step, spec } = this.state; - if (!Object.keys(spec).length && !this.props.initSupervisorId && !this.props.initTaskId) { - return ( -
- {this.renderInitStep()} -
- ); - } - + const { step } = this.state; return (
{this.renderStepNav()} + {step === 'welcome' && this.renderWelcomeStep()} {step === 'connect' && this.renderConnectStep()} {step === 'parser' && this.renderParserStep()} {step === 'timestamp' && this.renderTimestampStep()} @@ -414,7 +422,7 @@ export class LoadDataView extends React.PureComponent this.updateStep(s)} - icon={s === 'json-spec' && IconNames.MANUALLY_ENTERED_DATA} + icon={s === 'spec' && IconNames.MANUALLY_ENTERED_DATA} text={VIEW_TITLE[s]} /> ))} @@ -484,79 +492,234 @@ export class LoadDataView extends React.PureComponent { - this.updateStep('connect'); - }, 10); - } - - renderIngestionCard(title: string, comboType: IngestionComboType, requiredModule?: string) { - const { overlordModules } = this.state; + renderIngestionCard(comboType: IngestionComboTypeWithExtra) { + const { overlordModules, selectedComboType } = this.state; if (!overlordModules) return null; + const requiredModule = getRequiredModule(comboType); const goodToGo = !requiredModule || overlordModules.includes(requiredModule); return ( { - if (goodToGo) { - this.initWith(comboType); - } else { - this.setState({ - overlordModuleNeededMessage: `${title} ingestion requires the '${requiredModule}' to be loaded.`, - }); - } + this.setState({ selectedComboType: selectedComboType !== comboType ? comboType : null }); }} > - {title} + +

{getIngestionTitle(comboType)}

 ); } - renderInitStep() { - const { goToTask } = this.props; - const { overlordModuleNeededMessage } = this.state; + renderWelcomeStep() { + const { spec } = this.state; return ( <> -
Please specify where your raw data is located
- -
- {this.renderIngestionCard('Apache Kafka', 'kafka', 'druid-kafka-indexing-service')} - {this.renderIngestionCard('AWS Kinesis', 'kinesis', 'druid-kinesis-indexing-service')} - {this.renderIngestionCard('HTTP(s)', 'index:http')} - {this.renderIngestionCard('AWS S3', 'index:static-s3', 'druid-s3-extensions')} - {this.renderIngestionCard( - 'Google Cloud Storage', - 'index:static-google-blobstore', - 'druid-google-extensions', - )} - {this.renderIngestionCard('Local disk', 'index:local')} - goToTask(null, 'supervisor')}> - Other (streaming) - - goToTask(null, 'task')}> - Other (batch) - +
+ {this.renderIngestionCard('kafka')} + {this.renderIngestionCard('kinesis')} + {this.renderIngestionCard('index:static-s3')} + {this.renderIngestionCard('index:static-google-blobstore')} + {this.renderIngestionCard('hadoop')} + {this.renderIngestionCard('index:http')} + {this.renderIngestionCard('index:local')} + {/* this.renderIngestionCard('example') */} + {this.renderIngestionCard('other')} +
+
+ {this.renderWelcomeStepMessage()} + {this.renderWelcomeStepControls()} + {!isEmptyIngestionSpec(spec) && ( +
- - this.setState({ overlordModuleNeededMessage: null })} - > -

{overlordModuleNeededMessage}

- ); } + renderWelcomeStepMessage() { + const { selectedComboType } = this.state; + + if (!selectedComboType) { + return

Please specify where your raw data is located

; + } + + const issue = this.selectedIngestionTypeIssue(); + if (issue) return issue; + + switch (selectedComboType) { + case 'index:http': + return ( + <> +

Load data accessible through HTTP(s).

+

+ Data must be in a text format and the HTTP(s) endpoint must be reachable by every + Druid process in the cluster. +

+ + ); + + case 'index:local': + return ( + <> +

+ Recommended only in single server deployments. +

+

Load data directly from a local file.

+

+ Files must be in a text format and must be accessible to all the Druid processes in + the cluster. +

+ + ); + + case 'index:static-s3': + return

Load text based data from Amazon S3.

; + + case 'index:static-google-blobstore': + return

Load text based data from the Google Blobstore.

; + + case 'kafka': + return

Load streaming data in real-time from Apache Kafka.

; + + case 'kinesis': + return

Load streaming data in real-time from Amazon Kinesis.

; + + case 'hadoop': + return ( + <> +

+ Data loader support coming soon! +

+

+ You can not ingest data from HDFS via the data loader at this time, however you can + ingest it through a Druid task. +

+

+ Please follow{' '} + + the hadoop docs + {' '} + and submit a JSON spec to start the task. +

+ + ); + + case 'example': + return

Pick one of these examples to get you started.

; + + case 'other': + return ( +

+ If you do not see your source of raw data here, you can try to ingest it by submitting a{' '} + + JSON task or supervisor spec + + . +

+ ); + + default: + return

Unknown ingestion type.

; + } + } + + renderWelcomeStepControls() { + const { goToTask } = this.props; + const { spec, selectedComboType } = this.state; + + const issue = this.selectedIngestionTypeIssue(); + if (issue) return null; + + switch (selectedComboType) { + case 'index:http': + case 'index:local': + case 'index:static-s3': + case 'index:static-google-blobstore': + case 'kafka': + case 'kinesis': + return ( + +
+ {!isEmptyIngestionSpec(spec) && ( +