Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update web-console data loader to support unified s3 and google input sources (#8994) 

* update web-console data loader to support unified s3 and google input source

* fixes

* add placeholder for objects

* only show objects if it already exists
This commit is contained in:
Clint Wylie 2019-12-06 07:25:26 -08:00 committed by Vadim Ogievetsky
parent ca2a7a1f08
commit cefcfe26dc
1 changed files with 82 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
web-console/src/utils/ingestion-spec.tsx
View File

@ -960,7 +960,7 @@ export interface InputSource {
filter?: any;
uris?: string[];
prefixes?: string[];
blobs?: { bucket: string; path: string }[];
objects?: { bucket: string; path: string }[];
fetchTimeout?: number;
// druid
@ -1156,14 +1156,15 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
type: 'string-array',
placeholder: 's3://your-bucket/some-file1.ext, s3://your-bucket/some-file2.ext',
required: true,
defined: ioConfig => !deepGet(ioConfig, 'inputSource.prefixes'),
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
The full S3 URI of your file. To ingest from multiple URIs, use commas to separate
each individual URI.
</p>
<p>Either S3 URIs or S3 prefixes must be set.</p>
<p>Either S3 URIs or prefixes or objects must be set.</p>
</>
),
},
@ -1173,11 +1174,34 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
type: 'string-array',
placeholder: 's3://your-bucket/some-path1, s3://your-bucket/some-path2',
required: true,
defined: ioConfig => !deepGet(ioConfig, 'inputSource.uris'),
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>A list of paths (with bucket) where your files are stored.</p>
<p>Either S3 URIs or S3 prefixes must be set.</p>
<p>Either S3 URIs or prefixes or objects must be set.</p>
</>
),
},
{
name: 'inputSource.objects',
label: 'S3 objects',
type: 'json',
placeholder: '{"bucket":"your-bucket", "path":"some-file.ext"}',
required: true,
defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
JSON array of{' '}
<ExternalLink
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/s3.html`}
>
S3 Objects
</ExternalLink>
.
</p>
<p>Either S3 URIs or prefixes or objects must be set.</p>
</>
),
},
@ -1187,21 +1211,57 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
return [
inputSourceType,
{
name: 'inputSource.blobs',
label: 'Google blobs',
type: 'json',
name: 'inputSource.uris',
label: 'Google Cloud Storage URIs',
type: 'string-array',
placeholder: 'gs://your-bucket/some-file1.ext, gs://your-bucket/some-file2.ext',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
The full Google Cloud Storage URI of your file. To ingest from multiple URIs, use
commas to separate each individual URI.
</p>
<p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
</>
),
},
{
name: 'inputSource.prefixes',
label: 'Google Cloud Storage prefixes',
type: 'string-array',
placeholder: 'gs://your-bucket/some-path1, gs://your-bucket/some-path2',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>A list of paths (with bucket) where your files are stored.</p>
<p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
</>
),
},
{
name: 'inputSource.objects',
label: 'Google Cloud Storage objects',
type: 'json',
placeholder: '{"bucket":"your-bucket", "path":"some-file.ext"}',
required: true,
defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
JSON array of{' '}
<ExternalLink
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-contrib/google.html`}
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/google.html`}
>
Google Blobs
Google Cloud Storage Objects
</ExternalLink>
.
</p>
<p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
</>
),
},
@ -1362,14 +1422,13 @@ function issueWithInputSource(inputSource: InputSource | undefined): string | un
break;
case 's3':
if (!nonEmptyArray(inputSource.uris) && !nonEmptyArray(inputSource.prefixes)) {
return 'must have at least one uri or prefix';
}
break;
case 'google':
if (!nonEmptyArray(inputSource.blobs)) {
return 'must have at least one blob';
if (
!nonEmptyArray(inputSource.uris) &&
!nonEmptyArray(inputSource.prefixes) &&
!nonEmptyArray(inputSource.objects)
) {
return 'must have at least one uri or prefix or object';
}
break;
@ -1755,9 +1814,11 @@ export function guessDataSourceName(spec: IngestionSpec): string | undefined {
}
case 's3':
const s3Path =
case 'google':
const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
const uriPath =
(inputSource.uris || EMPTY_ARRAY)[0] || (inputSource.prefixes || EMPTY_ARRAY)[0];
return s3Path ? filenameFromPath(s3Path) : undefined;
return actualPath ? actualPath.path : uriPath ? filenameFromPath(uriPath) : undefined;
case 'http':
return Array.isArray(inputSource.uris)
@ -2556,6 +2617,7 @@ export function upgradeSpec(spec: any): any {
case 'static-google-blobstore':
deepSet(spec, 'ioConfig.firehose.type', 'google');
deepMove(spec, 'ioConfig.firehose.blobs', 'ioConfig.firehose.objects');
break;
}
@ -2593,6 +2655,7 @@ export function downgradeSpec(spec: any): any {
case 'google':
deepSet(spec, 'ioConfig.firehose.type', 'static-google-blobstore');
deepMove(spec, 'ioConfig.firehose.objects', 'ioConfig.firehose.blobs');
break;
}
}