mirror of https://github.com/apache/druid.git
Web console: Power up the data loader init step (#7947)
* Power up the data loader init step * update snapshot * normalize spec * allow deselect * added HDFS tile * update border style * text updates * goodies * new reset icon
This commit is contained in:
Vadim Ogievetsky
Fangjin Yang
parent
bc1413e4e3
commit
d677c83ce4
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
|
|
@ -229,9 +229,9 @@
|
|||
}
|
||||
},
|
||||
"@blueprintjs/core": {
|
||||
"version": "3.15.1",
|
||||
"resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.15.1.tgz",
|
||||
"integrity": "sha512-M8ltbqqlMZuZ6SEuqo/3Fr59ZcUfd8Er7ocbm7EACVfRW7dRhOCd/TKkf2kfICNtCDwznwXk0iAePLXZhUGtQg==",
|
||||
"version": "3.16.2",
|
||||
"resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.16.2.tgz",
|
||||
"integrity": "sha512-u+mSITWaNDwbdaPrbKx9XyxGsF4725SCAidWjd367ysX7AxCo4PK4SsFQVfXNylXpVWHQhJZekuo7+hdksc9lA==",
|
||||
"requires": {
|
||||
"@blueprintjs/icons": "^3.8.0",
|
||||
"@types/dom4": "^2.0.1",
|
||||
|
|
@ -246,9 +246,9 @@
|
|||
}
|
||||
},
|
||||
"@blueprintjs/icons": {
|
||||
"version": "3.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.8.0.tgz",
|
||||
"integrity": "sha512-yHaRQ3vfV9Gf3foZ4ONtxddz+u5ufkHqHj8Ia5VhPbFgG4el+cPdmsGGIIM72rgKS1KQa5Ay+ggjpByUlXvrKg==",
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.9.0.tgz",
|
||||
"integrity": "sha512-kq1Bh6PtOF4PcuxcDme8NmnSlkfO0IV89FriZGo6zSA1+OOzSwzvoKqa6S7vJe8xCPPLO5r7lE9AjeOuGeH97g==",
|
||||
"requires": {
|
||||
"classnames": "^2.2",
|
||||
"tslib": "^1.9.0"
|
||||
|
|
|
|||
|
|
@ -50,7 +50,8 @@
|
|||
"stylelint": "stylelint 'src/**/*.scss'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@blueprintjs/core": "^3.15.1",
|
||||
"@blueprintjs/core": "^3.16.2",
|
||||
"@blueprintjs/icons": "^3.9.0",
|
||||
"@types/memoize-one": "^4.1.1",
|
||||
"axios": "^0.19.0",
|
||||
"brace": "^0.11.1",
|
||||
|
|
|
|||
|
|
@ -25,3 +25,4 @@ cp -r coordinator-console "$1"
|
|||
cp -r old-console "$1"
|
||||
cp -r pages "$1"
|
||||
cp -r public "$1"
|
||||
cp -r assets "$1"
|
||||
|
|
|
|||
|
|
@ -34,27 +34,31 @@ const readDoc = async () => {
|
|||
if (functionMatch) {
|
||||
functionDocs.push({
|
||||
syntax: functionMatch[1],
|
||||
description: functionMatch[2]
|
||||
})
|
||||
description: functionMatch[2],
|
||||
});
|
||||
}
|
||||
|
||||
const dataTypeMatch = line.match(/^\|([A-Z]+)\|([A-Z]+)\|(.*)\|(.*)\|$/);
|
||||
if (dataTypeMatch) {
|
||||
dataTypeDocs.push({
|
||||
syntax: dataTypeMatch[1],
|
||||
description: dataTypeMatch[4] || `Druid runtime type: ${dataTypeMatch[2]}`
|
||||
})
|
||||
description: dataTypeMatch[4] || `Druid runtime type: ${dataTypeMatch[2]}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure there are at least 10 functions for sanity
|
||||
if (functionDocs.length < 10) {
|
||||
throw new Error(`Did not find enough function entries did the structure of '${readfile}' change? (found ${functionDocs.length})`);
|
||||
throw new Error(
|
||||
`Did not find enough function entries did the structure of '${readfile}' change? (found ${functionDocs.length})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Make sure there are at least 5 data types for sanity
|
||||
if (dataTypeDocs.length < 10) {
|
||||
throw new Error(`Did not find enough data type entries did the structure of '${readfile}' change? (found ${dataTypeDocs.length})`);
|
||||
throw new Error(
|
||||
`Did not find enough data type entries did the structure of '${readfile}' change? (found ${dataTypeDocs.length})`,
|
||||
);
|
||||
}
|
||||
|
||||
const content = `/*
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ export interface IngestionSpec {
|
|||
tuningConfig?: TuningConfig;
|
||||
}
|
||||
|
||||
export function isEmptyIngestionSpec(spec: IngestionSpec) {
|
||||
return Object.keys(spec).length === 0;
|
||||
}
|
||||
|
||||
export type IngestionType = 'kafka' | 'kinesis' | 'index_hadoop' | 'index' | 'index_parallel';
|
||||
|
||||
// A combination of IngestionType and firehose
|
||||
|
|
@ -48,6 +52,9 @@ export type IngestionComboType =
|
|||
| 'index:static-s3'
|
||||
| 'index:static-google-blobstore';
|
||||
|
||||
// Some extra values that can be selected in the initial screen
|
||||
export type IngestionComboTypeWithExtra = IngestionComboType | 'hadoop' | 'example' | 'other';
|
||||
|
||||
function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): string {
|
||||
switch (ingestionType) {
|
||||
case 'kafka':
|
||||
|
|
@ -87,6 +94,65 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType |
|
|||
return null;
|
||||
}
|
||||
|
||||
export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): string {
|
||||
switch (ingestionType) {
|
||||
case 'index:local':
|
||||
return 'Local disk';
|
||||
|
||||
case 'index:http':
|
||||
return 'HTTP(s)';
|
||||
|
||||
case 'index:static-s3':
|
||||
return 'Amazon S3';
|
||||
|
||||
case 'index:static-google-blobstore':
|
||||
return 'Google Cloud Storage';
|
||||
|
||||
case 'kafka':
|
||||
return 'Apache Kafka';
|
||||
|
||||
case 'kinesis':
|
||||
return 'Amazon Kinesis';
|
||||
|
||||
case 'hadoop':
|
||||
return 'HDFS';
|
||||
|
||||
case 'example':
|
||||
return 'Example data';
|
||||
|
||||
case 'other':
|
||||
return 'Other';
|
||||
|
||||
default:
|
||||
return 'Unknown ingestion';
|
||||
}
|
||||
}
|
||||
|
||||
export function getIngestionImage(ingestionType: IngestionComboTypeWithExtra): string {
|
||||
const parts = ingestionType.split(':');
|
||||
if (parts.length === 2) return parts[1];
|
||||
return ingestionType;
|
||||
}
|
||||
|
||||
export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): string | null {
|
||||
switch (ingestionType) {
|
||||
case 'index:static-s3':
|
||||
return 'druid-s3-extensions';
|
||||
|
||||
case 'index:static-google-blobstore':
|
||||
return 'druid-google-extensions';
|
||||
|
||||
case 'kafka':
|
||||
return 'druid-kafka-indexing-service';
|
||||
|
||||
case 'kinesis':
|
||||
return 'druid-kinesis-indexing-service';
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// --------------
|
||||
|
||||
export interface DataSchema {
|
||||
|
|
@ -138,7 +204,7 @@ export function getRollup(spec: IngestionSpec): boolean {
|
|||
return typeof specRollup === 'boolean' ? specRollup : true;
|
||||
}
|
||||
|
||||
export function getSpecType(spec: IngestionSpec): IngestionType | undefined {
|
||||
export function getSpecType(spec: Partial<IngestionSpec>): IngestionType | undefined {
|
||||
return (
|
||||
deepGet(spec, 'type') || deepGet(spec, 'ioConfig.type') || deepGet(spec, 'tuningConfig.type')
|
||||
);
|
||||
|
|
@ -158,13 +224,21 @@ export function changeParallel(spec: IngestionSpec, parallel: boolean): Ingestio
|
|||
* Make sure that the types are set in the root, ioConfig, and tuningConfig
|
||||
* @param spec
|
||||
*/
|
||||
export function normalizeSpecType(spec: IngestionSpec) {
|
||||
export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
|
||||
if (!spec || typeof spec !== 'object') {
|
||||
// This does not match the type of IngestionSpec but this dialog is robust enough to deal with anything but spec must be an object
|
||||
spec = {};
|
||||
}
|
||||
|
||||
// Make sure that if we actually get a task payload we extract the spec
|
||||
if (typeof (spec as any).spec === 'object') spec = (spec as any).spec;
|
||||
|
||||
const specType = getSpecType(spec);
|
||||
if (!specType) return spec;
|
||||
if (!specType) return spec as IngestionSpec;
|
||||
if (!deepGet(spec, 'type')) spec = deepSet(spec, 'type', specType);
|
||||
if (!deepGet(spec, 'ioConfig.type')) spec = deepSet(spec, 'ioConfig.type', specType);
|
||||
if (!deepGet(spec, 'tuningConfig.type')) spec = deepSet(spec, 'tuningConfig.type', specType);
|
||||
return spec;
|
||||
return spec as IngestionSpec;
|
||||
}
|
||||
|
||||
const PARSE_SPEC_FORM_FIELDS: Field<ParseSpec>[] = [
|
||||
|
|
@ -851,7 +925,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
|
|||
],
|
||||
info: (
|
||||
<>
|
||||
The AWS Kinesis stream endpoint for a region. You can find a list of endpoints{' '}
|
||||
The Amazon Kinesis stream endpoint for a region. You can find a list of endpoints{' '}
|
||||
<ExternalLink href="http://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region">
|
||||
here
|
||||
</ExternalLink>
|
||||
|
|
@ -1662,40 +1736,40 @@ export interface Bitmap {
|
|||
|
||||
// --------------
|
||||
|
||||
export function getBlankSpec(comboType: IngestionComboType): IngestionSpec {
|
||||
export function updateIngestionType(
|
||||
spec: IngestionSpec,
|
||||
comboType: IngestionComboType,
|
||||
): IngestionSpec {
|
||||
let [ingestionType, firehoseType] = comboType.split(':');
|
||||
if (ingestionType === 'index') ingestionType = 'index_parallel';
|
||||
const ioAndTuningConfigType = ingestionTypeToIoAndTuningConfigType(
|
||||
ingestionType as IngestionType,
|
||||
);
|
||||
|
||||
const granularitySpec: GranularitySpec = {
|
||||
type: 'uniform',
|
||||
segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR',
|
||||
queryGranularity: 'HOUR',
|
||||
};
|
||||
|
||||
const spec: IngestionSpec = {
|
||||
type: ingestionType,
|
||||
dataSchema: {
|
||||
dataSource: 'new-data-source',
|
||||
granularitySpec,
|
||||
},
|
||||
ioConfig: {
|
||||
type: ioAndTuningConfigType,
|
||||
},
|
||||
tuningConfig: {
|
||||
type: ioAndTuningConfigType,
|
||||
},
|
||||
} as any;
|
||||
let newSpec = spec;
|
||||
newSpec = deepSet(newSpec, 'type', ingestionType);
|
||||
newSpec = deepSet(newSpec, 'ioConfig.type', ioAndTuningConfigType);
|
||||
newSpec = deepSet(newSpec, 'tuningConfig.type', ioAndTuningConfigType);
|
||||
|
||||
if (firehoseType) {
|
||||
spec.ioConfig.firehose = {
|
||||
type: firehoseType,
|
||||
};
|
||||
newSpec = deepSet(newSpec, 'ioConfig.firehose', { type: firehoseType });
|
||||
}
|
||||
|
||||
return spec;
|
||||
if (!deepGet(spec, 'dataSchema.dataSource')) {
|
||||
newSpec = deepSet(newSpec, 'dataSchema.dataSource', 'new-data-source');
|
||||
}
|
||||
|
||||
if (!deepGet(spec, 'dataSchema.granularitySpec')) {
|
||||
const granularitySpec: GranularitySpec = {
|
||||
type: 'uniform',
|
||||
segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR',
|
||||
queryGranularity: 'HOUR',
|
||||
};
|
||||
|
||||
newSpec = deepSet(newSpec, 'dataSchema.granularitySpec', granularitySpec);
|
||||
}
|
||||
|
||||
return newSpec;
|
||||
}
|
||||
|
||||
export function fillParser(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
|
||||
|
|
|
|||
|
|
@ -2,41 +2,169 @@
|
|||
|
||||
exports[`load data view matches snapshot 1`] = `
|
||||
<div
|
||||
className="load-data-view app-view init"
|
||||
className="load-data-view app-view welcome"
|
||||
>
|
||||
<div
|
||||
className="intro"
|
||||
className="bp3-tabs step-nav"
|
||||
>
|
||||
Please specify where your raw data is located
|
||||
<div
|
||||
className="step-section"
|
||||
key="Connect and parse raw data"
|
||||
>
|
||||
<div
|
||||
className="step-nav-l1"
|
||||
>
|
||||
Connect and parse raw data
|
||||
</div>
|
||||
<Blueprint3.ButtonGroup
|
||||
className="step-nav-l2"
|
||||
>
|
||||
<Blueprint3.Button
|
||||
active={true}
|
||||
className="welcome"
|
||||
icon={false}
|
||||
key="welcome"
|
||||
onClick={[Function]}
|
||||
text="Start"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="connect"
|
||||
icon={false}
|
||||
key="connect"
|
||||
onClick={[Function]}
|
||||
text="Connect"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="parser"
|
||||
icon={false}
|
||||
key="parser"
|
||||
onClick={[Function]}
|
||||
text="Parse data"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="timestamp"
|
||||
icon={false}
|
||||
key="timestamp"
|
||||
onClick={[Function]}
|
||||
text="Parse time"
|
||||
/>
|
||||
</Blueprint3.ButtonGroup>
|
||||
</div>
|
||||
<div
|
||||
className="step-section"
|
||||
key="Transform and configure schema"
|
||||
>
|
||||
<div
|
||||
className="step-nav-l1"
|
||||
>
|
||||
Transform and configure schema
|
||||
</div>
|
||||
<Blueprint3.ButtonGroup
|
||||
className="step-nav-l2"
|
||||
>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="transform"
|
||||
icon={false}
|
||||
key="transform"
|
||||
onClick={[Function]}
|
||||
text="Transform"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="filter"
|
||||
icon={false}
|
||||
key="filter"
|
||||
onClick={[Function]}
|
||||
text="Filter"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="schema"
|
||||
icon={false}
|
||||
key="schema"
|
||||
onClick={[Function]}
|
||||
text="Configure schema"
|
||||
/>
|
||||
</Blueprint3.ButtonGroup>
|
||||
</div>
|
||||
<div
|
||||
className="step-section"
|
||||
key="Tune parameters"
|
||||
>
|
||||
<div
|
||||
className="step-nav-l1"
|
||||
>
|
||||
Tune parameters
|
||||
</div>
|
||||
<Blueprint3.ButtonGroup
|
||||
className="step-nav-l2"
|
||||
>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="partition"
|
||||
icon={false}
|
||||
key="partition"
|
||||
onClick={[Function]}
|
||||
text="Partition"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="tuning"
|
||||
icon={false}
|
||||
key="tuning"
|
||||
onClick={[Function]}
|
||||
text="Tune"
|
||||
/>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="publish"
|
||||
icon={false}
|
||||
key="publish"
|
||||
onClick={[Function]}
|
||||
text="Publish"
|
||||
/>
|
||||
</Blueprint3.ButtonGroup>
|
||||
</div>
|
||||
<div
|
||||
className="step-section"
|
||||
key="Verify and submit"
|
||||
>
|
||||
<div
|
||||
className="step-nav-l1"
|
||||
>
|
||||
Verify and submit
|
||||
</div>
|
||||
<Blueprint3.ButtonGroup
|
||||
className="step-nav-l2"
|
||||
>
|
||||
<Blueprint3.Button
|
||||
active={false}
|
||||
className="spec"
|
||||
icon="manually-entered-data"
|
||||
key="spec"
|
||||
onClick={[Function]}
|
||||
text="Edit JSON spec"
|
||||
/>
|
||||
</Blueprint3.ButtonGroup>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
className="cards"
|
||||
className="main"
|
||||
/>
|
||||
<div
|
||||
className="control"
|
||||
>
|
||||
<Blueprint3.Card
|
||||
elevation={0}
|
||||
interactive={true}
|
||||
onClick={[Function]}
|
||||
<Blueprint3.Callout
|
||||
className="intro"
|
||||
>
|
||||
Other (streaming)
|
||||
</Blueprint3.Card>
|
||||
<Blueprint3.Card
|
||||
elevation={0}
|
||||
interactive={true}
|
||||
onClick={[Function]}
|
||||
>
|
||||
Other (batch)
|
||||
</Blueprint3.Card>
|
||||
<p>
|
||||
Please specify where your raw data is located
|
||||
</p>
|
||||
</Blueprint3.Callout>
|
||||
</div>
|
||||
<Blueprint3.Alert
|
||||
canEscapeKeyCancel={false}
|
||||
canOutsideClickCancel={false}
|
||||
confirmButtonText="Close"
|
||||
icon="warning-sign"
|
||||
intent="warning"
|
||||
isOpen={false}
|
||||
onConfirm={[Function]}
|
||||
>
|
||||
<p />
|
||||
</Blueprint3.Alert>
|
||||
</div>
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -27,32 +27,45 @@
|
|||
'main ctrl'
|
||||
'main next';
|
||||
|
||||
&.init {
|
||||
display: block;
|
||||
&.welcome {
|
||||
.main {
|
||||
margin-left: -10px;
|
||||
|
||||
& > * {
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
|
||||
.intro {
|
||||
font-size: 20px;
|
||||
}
|
||||
|
||||
.cards {
|
||||
.bp3-card {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
vertical-align: top;
|
||||
width: 250px;
|
||||
height: 140px;
|
||||
margin-right: 15px;
|
||||
margin-left: 15px;
|
||||
margin-bottom: 15px;
|
||||
font-size: 24px;
|
||||
font-size: 16px;
|
||||
text-align: center;
|
||||
padding-top: 47px;
|
||||
|
||||
& > * {
|
||||
user-select: none;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
&.active::after {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
content: '';
|
||||
border: 2px solid #48aff0;
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
&.disabled {
|
||||
opacity: 0.4;
|
||||
}
|
||||
|
||||
img {
|
||||
width: 100px;
|
||||
display: inline-block;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -192,7 +205,7 @@
|
|||
text-align: right;
|
||||
padding: 0 5px;
|
||||
|
||||
.prev {
|
||||
.left {
|
||||
float: left;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import {
|
|||
Card,
|
||||
Classes,
|
||||
Code,
|
||||
Elevation,
|
||||
FormGroup,
|
||||
H5,
|
||||
HTMLSelect,
|
||||
|
|
@ -50,6 +51,7 @@ import {
|
|||
} from '../../components';
|
||||
import { AsyncActionDialog } from '../../dialogs';
|
||||
import { AppToaster } from '../../singletons/toaster';
|
||||
import { UrlBaser } from '../../singletons/url-baser';
|
||||
import {
|
||||
filterMap,
|
||||
getDruidErrorMessage,
|
||||
|
|
@ -72,18 +74,20 @@ import {
|
|||
fillDataSourceName,
|
||||
fillParser,
|
||||
FlattenField,
|
||||
getBlankSpec,
|
||||
getDimensionMode,
|
||||
getDimensionSpecFormFields,
|
||||
getEmptyTimestampSpec,
|
||||
getFilterFormFields,
|
||||
getFlattenFieldFormFields,
|
||||
getIngestionComboType,
|
||||
getIngestionImage,
|
||||
getIngestionTitle,
|
||||
getIoConfigFormFields,
|
||||
getIoConfigTuningFormFields,
|
||||
getMetricSpecFormFields,
|
||||
getParseSpecFormFields,
|
||||
getPartitionRelatedTuningSpecFormFields,
|
||||
getRequiredModule,
|
||||
getRollup,
|
||||
getSpecType,
|
||||
getTimestampSpecFormFields,
|
||||
|
|
@ -91,16 +95,17 @@ import {
|
|||
getTuningSpecFormFields,
|
||||
GranularitySpec,
|
||||
hasParallelAbility,
|
||||
IngestionComboType,
|
||||
IngestionComboTypeWithExtra,
|
||||
IngestionSpec,
|
||||
IoConfig,
|
||||
isColumnTimestampSpec,
|
||||
isEmptyIngestionSpec,
|
||||
isParallel,
|
||||
issueWithIoConfig,
|
||||
issueWithParser,
|
||||
joinFilter,
|
||||
MetricSpec,
|
||||
normalizeSpecType,
|
||||
normalizeSpec,
|
||||
Parser,
|
||||
ParseSpec,
|
||||
parseSpecHasFlatten,
|
||||
|
|
@ -108,6 +113,7 @@ import {
|
|||
TimestampSpec,
|
||||
Transform,
|
||||
TuningConfig,
|
||||
updateIngestionType,
|
||||
} from '../../utils/ingestion-spec';
|
||||
import { deepDelete, deepGet, deepSet } from '../../utils/object-change';
|
||||
import {
|
||||
|
|
@ -163,6 +169,7 @@ function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec {
|
|||
}
|
||||
|
||||
type Step =
|
||||
| 'welcome'
|
||||
| 'connect'
|
||||
| 'parser'
|
||||
| 'timestamp'
|
||||
|
|
@ -172,9 +179,11 @@ type Step =
|
|||
| 'partition'
|
||||
| 'tuning'
|
||||
| 'publish'
|
||||
| 'json-spec'
|
||||
| 'spec'
|
||||
| 'loading';
|
||||
|
||||
const STEPS: Step[] = [
|
||||
'welcome',
|
||||
'connect',
|
||||
'parser',
|
||||
'timestamp',
|
||||
|
|
@ -184,18 +193,19 @@ const STEPS: Step[] = [
|
|||
'partition',
|
||||
'tuning',
|
||||
'publish',
|
||||
'json-spec',
|
||||
'spec',
|
||||
'loading',
|
||||
];
|
||||
|
||||
const SECTIONS: { name: string; steps: Step[] }[] = [
|
||||
{ name: 'Connect and parse raw data', steps: ['connect', 'parser', 'timestamp'] },
|
||||
{ name: 'Connect and parse raw data', steps: ['welcome', 'connect', 'parser', 'timestamp'] },
|
||||
{ name: 'Transform and configure schema', steps: ['transform', 'filter', 'schema'] },
|
||||
{ name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] },
|
||||
{ name: 'Verify and submit', steps: ['json-spec'] },
|
||||
{ name: 'Verify and submit', steps: ['spec'] },
|
||||
];
|
||||
|
||||
const VIEW_TITLE: Record<Step, string> = {
|
||||
welcome: 'Start',
|
||||
connect: 'Connect',
|
||||
parser: 'Parse data',
|
||||
timestamp: 'Parse time',
|
||||
|
|
@ -205,7 +215,7 @@ const VIEW_TITLE: Record<Step, string> = {
|
|||
partition: 'Partition',
|
||||
tuning: 'Tune',
|
||||
publish: 'Publish',
|
||||
'json-spec': 'Edit JSON spec',
|
||||
spec: 'Edit JSON spec',
|
||||
loading: 'Loading',
|
||||
};
|
||||
|
||||
|
|
@ -224,9 +234,11 @@ export interface LoadDataViewState {
|
|||
newRollup: boolean | null;
|
||||
newDimensionMode: DimensionMode | null;
|
||||
|
||||
// general
|
||||
// welcome
|
||||
overlordModules: string[] | null;
|
||||
overlordModuleNeededMessage: string | null;
|
||||
selectedComboType: IngestionComboTypeWithExtra | null;
|
||||
|
||||
// general
|
||||
sampleStrategy: SampleStrategy;
|
||||
columnFilter: string;
|
||||
specialColumnsOnly: boolean;
|
||||
|
|
@ -278,7 +290,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
let spec = parseJson(String(localStorageGet(LocalStorageKeys.INGESTION_SPEC)));
|
||||
if (!spec || typeof spec !== 'object') spec = {};
|
||||
this.state = {
|
||||
step: 'connect',
|
||||
step: 'welcome',
|
||||
spec,
|
||||
cacheKey: undefined,
|
||||
|
||||
|
|
@ -287,9 +299,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
newRollup: null,
|
||||
newDimensionMode: null,
|
||||
|
||||
// general
|
||||
// welcome
|
||||
overlordModules: null,
|
||||
overlordModuleNeededMessage: null,
|
||||
selectedComboType: null,
|
||||
|
||||
// general
|
||||
sampleStrategy: 'start',
|
||||
columnFilter: '',
|
||||
specialColumnsOnly: false,
|
||||
|
|
@ -329,6 +343,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
}
|
||||
|
||||
componentDidMount(): void {
|
||||
const { spec } = this.state;
|
||||
|
||||
this.getOverlordModules();
|
||||
if (this.props.initTaskId) {
|
||||
this.updateStep('loading');
|
||||
|
|
@ -336,6 +352,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
} else if (this.props.initSupervisorId) {
|
||||
this.updateStep('loading');
|
||||
this.getSupervisorJson();
|
||||
} else if (isEmptyIngestionSpec(spec)) {
|
||||
this.updateStep('welcome');
|
||||
} else {
|
||||
this.updateStep('connect');
|
||||
}
|
||||
|
|
@ -380,28 +398,18 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
}
|
||||
|
||||
private updateSpec = (newSpec: IngestionSpec) => {
|
||||
if (!newSpec || typeof newSpec !== 'object') {
|
||||
// This does not match the type of IngestionSpec but this dialog is robust enough to deal with anything but spec must be an object
|
||||
newSpec = {} as any;
|
||||
}
|
||||
newSpec = normalizeSpec(newSpec);
|
||||
this.setState({ spec: newSpec });
|
||||
localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(newSpec));
|
||||
};
|
||||
|
||||
render() {
|
||||
const { step, spec } = this.state;
|
||||
if (!Object.keys(spec).length && !this.props.initSupervisorId && !this.props.initTaskId) {
|
||||
return (
|
||||
<div className={classNames('load-data-view', 'app-view', 'init')}>
|
||||
{this.renderInitStep()}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const { step } = this.state;
|
||||
return (
|
||||
<div className={classNames('load-data-view', 'app-view', step)}>
|
||||
{this.renderStepNav()}
|
||||
|
||||
{step === 'welcome' && this.renderWelcomeStep()}
|
||||
{step === 'connect' && this.renderConnectStep()}
|
||||
{step === 'parser' && this.renderParserStep()}
|
||||
{step === 'timestamp' && this.renderTimestampStep()}
|
||||
|
|
@ -414,7 +422,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
{step === 'tuning' && this.renderTuningStep()}
|
||||
{step === 'publish' && this.renderPublishStep()}
|
||||
|
||||
{step === 'json-spec' && this.renderJsonSpecStep()}
|
||||
{step === 'spec' && this.renderSpecStep()}
|
||||
{step === 'loading' && this.renderLoading()}
|
||||
|
||||
{this.renderResetConfirm()}
|
||||
|
|
@ -437,7 +445,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
key={s}
|
||||
active={s === step}
|
||||
onClick={() => this.updateStep(s)}
|
||||
icon={s === 'json-spec' && IconNames.MANUALLY_ENTERED_DATA}
|
||||
icon={s === 'spec' && IconNames.MANUALLY_ENTERED_DATA}
|
||||
text={VIEW_TITLE[s]}
|
||||
/>
|
||||
))}
|
||||
|
|
@ -484,79 +492,234 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
|
||||
// ==================================================================
|
||||
|
||||
initWith(comboType: IngestionComboType) {
|
||||
this.setState({
|
||||
spec: getBlankSpec(comboType),
|
||||
});
|
||||
setTimeout(() => {
|
||||
this.updateStep('connect');
|
||||
}, 10);
|
||||
}
|
||||
|
||||
renderIngestionCard(title: string, comboType: IngestionComboType, requiredModule?: string) {
|
||||
const { overlordModules } = this.state;
|
||||
renderIngestionCard(comboType: IngestionComboTypeWithExtra) {
|
||||
const { overlordModules, selectedComboType } = this.state;
|
||||
if (!overlordModules) return null;
|
||||
const requiredModule = getRequiredModule(comboType);
|
||||
const goodToGo = !requiredModule || overlordModules.includes(requiredModule);
|
||||
|
||||
return (
|
||||
<Card
|
||||
className={classNames({ disabled: !goodToGo })}
|
||||
className={classNames({ disabled: !goodToGo, active: selectedComboType === comboType })}
|
||||
interactive
|
||||
onClick={() => {
|
||||
if (goodToGo) {
|
||||
this.initWith(comboType);
|
||||
} else {
|
||||
this.setState({
|
||||
overlordModuleNeededMessage: `${title} ingestion requires the '${requiredModule}' to be loaded.`,
|
||||
});
|
||||
}
|
||||
this.setState({ selectedComboType: selectedComboType !== comboType ? comboType : null });
|
||||
}}
|
||||
>
|
||||
{title}
|
||||
<img src={UrlBaser.base(`/assets/${getIngestionImage(comboType)}.png`)} />
|
||||
<p>{getIngestionTitle(comboType)}</p>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
renderInitStep() {
|
||||
const { goToTask } = this.props;
|
||||
const { overlordModuleNeededMessage } = this.state;
|
||||
renderWelcomeStep() {
|
||||
const { spec } = this.state;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="intro">Please specify where your raw data is located</div>
|
||||
|
||||
<div className="cards">
|
||||
{this.renderIngestionCard('Apache Kafka', 'kafka', 'druid-kafka-indexing-service')}
|
||||
{this.renderIngestionCard('AWS Kinesis', 'kinesis', 'druid-kinesis-indexing-service')}
|
||||
{this.renderIngestionCard('HTTP(s)', 'index:http')}
|
||||
{this.renderIngestionCard('AWS S3', 'index:static-s3', 'druid-s3-extensions')}
|
||||
{this.renderIngestionCard(
|
||||
'Google Cloud Storage',
|
||||
'index:static-google-blobstore',
|
||||
'druid-google-extensions',
|
||||
)}
|
||||
{this.renderIngestionCard('Local disk', 'index:local')}
|
||||
<Card interactive onClick={() => goToTask(null, 'supervisor')}>
|
||||
Other (streaming)
|
||||
</Card>
|
||||
<Card interactive onClick={() => goToTask(null, 'task')}>
|
||||
Other (batch)
|
||||
</Card>
|
||||
<div className="main">
|
||||
{this.renderIngestionCard('kafka')}
|
||||
{this.renderIngestionCard('kinesis')}
|
||||
{this.renderIngestionCard('index:static-s3')}
|
||||
{this.renderIngestionCard('index:static-google-blobstore')}
|
||||
{this.renderIngestionCard('hadoop')}
|
||||
{this.renderIngestionCard('index:http')}
|
||||
{this.renderIngestionCard('index:local')}
|
||||
{/* this.renderIngestionCard('example') */}
|
||||
{this.renderIngestionCard('other')}
|
||||
</div>
|
||||
<div className="control">
|
||||
<Callout className="intro">{this.renderWelcomeStepMessage()}</Callout>
|
||||
{this.renderWelcomeStepControls()}
|
||||
{!isEmptyIngestionSpec(spec) && (
|
||||
<Button icon={IconNames.RESET} text="Reset spec" onClick={this.handleResetConfirm} />
|
||||
)}
|
||||
</div>
|
||||
|
||||
<Alert
|
||||
icon={IconNames.WARNING_SIGN}
|
||||
intent={Intent.WARNING}
|
||||
isOpen={Boolean(overlordModuleNeededMessage)}
|
||||
confirmButtonText="Close"
|
||||
onConfirm={() => this.setState({ overlordModuleNeededMessage: null })}
|
||||
>
|
||||
<p>{overlordModuleNeededMessage}</p>
|
||||
</Alert>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
renderWelcomeStepMessage() {
|
||||
const { selectedComboType } = this.state;
|
||||
|
||||
if (!selectedComboType) {
|
||||
return <p>Please specify where your raw data is located</p>;
|
||||
}
|
||||
|
||||
const issue = this.selectedIngestionTypeIssue();
|
||||
if (issue) return issue;
|
||||
|
||||
switch (selectedComboType) {
|
||||
case 'index:http':
|
||||
return (
|
||||
<>
|
||||
<p>Load data accessible through HTTP(s).</p>
|
||||
<p>
|
||||
Data must be in a text format and the HTTP(s) endpoint must be reachable by every
|
||||
Druid process in the cluster.
|
||||
</p>
|
||||
</>
|
||||
);
|
||||
|
||||
case 'index:local':
|
||||
return (
|
||||
<>
|
||||
<p>
|
||||
<em>Recommended only in single server deployments.</em>
|
||||
</p>
|
||||
<p>Load data directly from a local file.</p>
|
||||
<p>
|
||||
Files must be in a text format and must be accessible to all the Druid processes in
|
||||
the cluster.
|
||||
</p>
|
||||
</>
|
||||
);
|
||||
|
||||
case 'index:static-s3':
|
||||
return <p>Load text based data from Amazon S3.</p>;
|
||||
|
||||
case 'index:static-google-blobstore':
|
||||
return <p>Load text based data from the Google Blobstore.</p>;
|
||||
|
||||
case 'kafka':
|
||||
return <p>Load streaming data in real-time from Apache Kafka.</p>;
|
||||
|
||||
case 'kinesis':
|
||||
return <p>Load streaming data in real-time from Amazon Kinesis.</p>;
|
||||
|
||||
case 'hadoop':
|
||||
return (
|
||||
<>
|
||||
<p>
|
||||
<em>Data loader support coming soon!</em>
|
||||
</p>
|
||||
<p>
|
||||
You can not ingest data from HDFS via the data loader at this time, however you can
|
||||
ingest it through a Druid task.
|
||||
</p>
|
||||
<p>
|
||||
Please follow{' '}
|
||||
<ExternalLink href="https://druid.apache.org/docs/latest/ingestion/hadoop.html">
|
||||
the hadoop docs
|
||||
</ExternalLink>{' '}
|
||||
and submit a JSON spec to start the task.
|
||||
</p>
|
||||
</>
|
||||
);
|
||||
|
||||
case 'example':
|
||||
return <p>Pick one of these examples to get you started.</p>;
|
||||
|
||||
case 'other':
|
||||
return (
|
||||
<p>
|
||||
If you do not see your source of raw data here, you can try to ingest it by submitting a{' '}
|
||||
<ExternalLink href="https://druid.apache.org/docs/latest/ingestion/index.html">
|
||||
JSON task or supervisor spec
|
||||
</ExternalLink>
|
||||
.
|
||||
</p>
|
||||
);
|
||||
|
||||
default:
|
||||
return <p>Unknown ingestion type.</p>;
|
||||
}
|
||||
}
|
||||
|
||||
renderWelcomeStepControls() {
|
||||
const { goToTask } = this.props;
|
||||
const { spec, selectedComboType } = this.state;
|
||||
|
||||
const issue = this.selectedIngestionTypeIssue();
|
||||
if (issue) return null;
|
||||
|
||||
switch (selectedComboType) {
|
||||
case 'index:http':
|
||||
case 'index:local':
|
||||
case 'index:static-s3':
|
||||
case 'index:static-google-blobstore':
|
||||
case 'kafka':
|
||||
case 'kinesis':
|
||||
return (
|
||||
<FormGroup>
|
||||
<Button
|
||||
text="Connect data"
|
||||
rightIcon={IconNames.ARROW_RIGHT}
|
||||
onClick={() => {
|
||||
this.setState({
|
||||
spec: updateIngestionType(spec, selectedComboType as any),
|
||||
});
|
||||
setTimeout(() => {
|
||||
this.updateStep('connect');
|
||||
}, 10);
|
||||
}}
|
||||
intent={Intent.PRIMARY}
|
||||
/>
|
||||
</FormGroup>
|
||||
);
|
||||
|
||||
case 'hadoop':
|
||||
return (
|
||||
<FormGroup>
|
||||
<Button
|
||||
text="Submit task"
|
||||
rightIcon={IconNames.ARROW_RIGHT}
|
||||
onClick={() => goToTask(null, 'task')}
|
||||
intent={Intent.PRIMARY}
|
||||
/>
|
||||
</FormGroup>
|
||||
);
|
||||
|
||||
case 'example':
|
||||
return null;
|
||||
|
||||
case 'other':
|
||||
return (
|
||||
<>
|
||||
<FormGroup>
|
||||
<Button
|
||||
text="Submit supervisor"
|
||||
rightIcon={IconNames.ARROW_RIGHT}
|
||||
onClick={() => goToTask(null, 'supervisor')}
|
||||
intent={Intent.PRIMARY}
|
||||
/>
|
||||
</FormGroup>
|
||||
<FormGroup>
|
||||
<Button
|
||||
text="Submit task"
|
||||
rightIcon={IconNames.ARROW_RIGHT}
|
||||
onClick={() => goToTask(null, 'task')}
|
||||
intent={Intent.PRIMARY}
|
||||
/>
|
||||
</FormGroup>
|
||||
</>
|
||||
);
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
selectedIngestionTypeIssue(): JSX.Element | null {
|
||||
const { selectedComboType, overlordModules } = this.state;
|
||||
if (!selectedComboType || !overlordModules) return null;
|
||||
|
||||
const requiredModule = getRequiredModule(selectedComboType);
|
||||
if (!requiredModule || overlordModules.includes(requiredModule)) return null;
|
||||
|
||||
return (
|
||||
<p>
|
||||
{`${getIngestionTitle(selectedComboType)} ingestion requires the `}
|
||||
<strong>{requiredModule}</strong>
|
||||
{` extension to be loaded.`}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
private handleResetConfirm = () => {
|
||||
this.setState({ showResetConfirm: true });
|
||||
};
|
||||
|
||||
renderResetConfirm() {
|
||||
const { showResetConfirm } = this.state;
|
||||
if (!showResetConfirm) return null;
|
||||
|
|
@ -713,8 +876,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
if (!inputQueryState.data) return;
|
||||
this.updateSpec(fillDataSourceName(fillParser(spec, inputQueryState.data)));
|
||||
},
|
||||
prevLabel: 'Restart',
|
||||
onPrevStep: () => this.setState({ showResetConfirm: true }),
|
||||
})}
|
||||
</>
|
||||
);
|
||||
|
|
@ -2398,8 +2559,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
|
||||
try {
|
||||
const resp = await axios.get(`/druid/indexer/v1/supervisor/${initSupervisorId}`);
|
||||
this.updateSpec(normalizeSpecType(resp.data));
|
||||
this.updateStep('json-spec');
|
||||
this.updateSpec(resp.data);
|
||||
this.updateStep('spec');
|
||||
} catch (e) {
|
||||
AppToaster.show({
|
||||
message: `Failed to get supervisor spec: ${getDruidErrorMessage(e)}`,
|
||||
|
|
@ -2413,8 +2574,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
|
||||
try {
|
||||
const resp = await axios.get(`/druid/indexer/v1/task/${initTaskId}`);
|
||||
this.updateSpec(normalizeSpecType(resp.data.payload.spec));
|
||||
this.updateStep('json-spec');
|
||||
this.updateSpec(resp.data.payload);
|
||||
this.updateStep('spec');
|
||||
} catch (e) {
|
||||
AppToaster.show({
|
||||
message: `Failed to get task spec: ${getDruidErrorMessage(e)}`,
|
||||
|
|
@ -2427,7 +2588,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
return <Loader loading />;
|
||||
}
|
||||
|
||||
renderJsonSpecStep() {
|
||||
renderSpecStep() {
|
||||
const { goToTask } = this.props;
|
||||
const { spec } = this.state;
|
||||
|
||||
|
|
@ -2438,7 +2599,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
value={spec}
|
||||
onChange={s => {
|
||||
if (!s) return;
|
||||
this.updateSpec(normalizeSpecType(s));
|
||||
this.updateSpec(s);
|
||||
}}
|
||||
height="100%"
|
||||
/>
|
||||
|
|
@ -2455,6 +2616,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
|
|||
</Callout>
|
||||
</div>
|
||||
<div className="next-bar">
|
||||
{!isEmptyIngestionSpec(spec) && (
|
||||
<Button
|
||||
className="left"
|
||||
icon={IconNames.RESET}
|
||||
text="Reset spec"
|
||||
onClick={this.handleResetConfirm}
|
||||
/>
|
||||
)}
|
||||
<Button
|
||||
text="Submit"
|
||||
intent={Intent.PRIMARY}
|
||||
|
|
|
|||
Loading…
Reference in New Issue