update web-console data loader to support unified s3 and google input sources (#8994)

* update web-console data loader to support unified s3 and google input source

* fixes

* add placeholder for objects

* only show objects if it already exists

web-console/src/utils/ingestion-spec.tsx

@@ -960,7 +960,7 @@ export interface InputSource {
   filter?: any;
   uris?: string[];
   prefixes?: string[];
-  blobs?: { bucket: string; path: string }[];
+  objects?: { bucket: string; path: string }[];
   fetchTimeout?: number;
 
   // druid
@@ -1156,14 +1156,15 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           type: 'string-array',
           placeholder: 's3://your-bucket/some-file1.ext, s3://your-bucket/some-file2.ext',
           required: true,
-          defined: ioConfig => !deepGet(ioConfig, 'inputSource.prefixes'),
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
           info: (
             <>
               <p>
                 The full S3 URI of your file. To ingest from multiple URIs, use commas to separate
                 each individual URI.
               </p>
-              <p>Either S3 URIs or S3 prefixes must be set.</p>
+              <p>Either S3 URIs or prefixes or objects must be set.</p>
             </>
           ),
         },
@@ -1173,11 +1174,34 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           type: 'string-array',
           placeholder: 's3://your-bucket/some-path1, s3://your-bucket/some-path2',
           required: true,
-          defined: ioConfig => !deepGet(ioConfig, 'inputSource.uris'),
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
           info: (
             <>
               <p>A list of paths (with bucket) where your files are stored.</p>
-              <p>Either S3 URIs or S3 prefixes must be set.</p>
+              <p>Either S3 URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.objects',
+          label: 'S3 objects',
+          type: 'json',
+          placeholder: '{"bucket":"your-bucket", "path":"some-file.ext"}',
+          required: true,
+          defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>
+                JSON array of{' '}
+                <ExternalLink
+                  href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/s3.html`}
+                >
+                  S3 Objects
+                </ExternalLink>
+                .
+              </p>
+              <p>Either S3 URIs or prefixes or objects must be set.</p>
             </>
           ),
         },
@@ -1187,21 +1211,57 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
       return [
         inputSourceType,
         {
-          name: 'inputSource.blobs',
+          name: 'inputSource.uris',
-          label: 'Google blobs',
+          label: 'Google Cloud Storage URIs',
-          type: 'json',
+          type: 'string-array',
+          placeholder: 'gs://your-bucket/some-file1.ext, gs://your-bucket/some-file2.ext',
           required: true,
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>
+                The full Google Cloud Storage URI of your file. To ingest from multiple URIs, use
+                commas to separate each individual URI.
+              </p>
+              <p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.prefixes',
+          label: 'Google Cloud Storage prefixes',
+          type: 'string-array',
+          placeholder: 'gs://your-bucket/some-path1, gs://your-bucket/some-path2',
+          required: true,
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>A list of paths (with bucket) where your files are stored.</p>
+              <p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.objects',
+          label: 'Google Cloud Storage objects',
+          type: 'json',
+          placeholder: '{"bucket":"your-bucket", "path":"some-file.ext"}',
+          required: true,
+          defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
           info: (
             <>
               <p>
                 JSON array of{' '}
                 <ExternalLink
-                  href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-contrib/google.html`}
+                  href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/google.html`}
                 >
-                  Google Blobs
+                  Google Cloud Storage Objects
                 </ExternalLink>
                 .
               </p>
+              <p>Either Google Cloud Storage URIs or prefixes or objects must be set.</p>
             </>
           ),
         },
@@ -1362,14 +1422,13 @@ function issueWithInputSource(inputSource: InputSource | undefined): string | un
       break;
 
     case 's3':
-      if (!nonEmptyArray(inputSource.uris) && !nonEmptyArray(inputSource.prefixes)) {
-        return 'must have at least one uri or prefix';
-      }
-      break;
-
     case 'google':
-      if (!nonEmptyArray(inputSource.blobs)) {
+      if (
-        return 'must have at least one blob';
+        !nonEmptyArray(inputSource.uris) &&
+        !nonEmptyArray(inputSource.prefixes) &&
+        !nonEmptyArray(inputSource.objects)
+      ) {
+        return 'must have at least one uri or prefix or object';
       }
       break;
 
@@ -1755,9 +1814,11 @@ export function guessDataSourceName(spec: IngestionSpec): string | undefined {
           }
 
         case 's3':
-          const s3Path =
+        case 'google':
+          const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
+          const uriPath =
             (inputSource.uris || EMPTY_ARRAY)[0] || (inputSource.prefixes || EMPTY_ARRAY)[0];
-          return s3Path ? filenameFromPath(s3Path) : undefined;
+          return actualPath ? actualPath.path : uriPath ? filenameFromPath(uriPath) : undefined;
 
         case 'http':
           return Array.isArray(inputSource.uris)
@@ -2556,6 +2617,7 @@ export function upgradeSpec(spec: any): any {
 
       case 'static-google-blobstore':
         deepSet(spec, 'ioConfig.firehose.type', 'google');
+        deepMove(spec, 'ioConfig.firehose.blobs', 'ioConfig.firehose.objects');
         break;
     }
 
@@ -2593,6 +2655,7 @@ export function downgradeSpec(spec: any): any {
 
       case 'google':
         deepSet(spec, 'ioConfig.firehose.type', 'static-google-blobstore');
+        deepMove(spec, 'ioConfig.firehose.objects', 'ioConfig.firehose.blobs');
         break;
     }
   }